Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/IR/IRBuilder.h
Warning:line 1673, column 28
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name OMPIRBuilder.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Frontend/OpenMP -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/CodeMetrics.h"
20#include "llvm/Analysis/LoopInfo.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ScalarEvolution.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/IR/CFG.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/GlobalVariable.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/IR/PassManager.h"
32#include "llvm/IR/Value.h"
33#include "llvm/MC/TargetRegistry.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Target/TargetMachine.h"
36#include "llvm/Target/TargetOptions.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/Cloning.h"
39#include "llvm/Transforms/Utils/CodeExtractor.h"
40#include "llvm/Transforms/Utils/LoopPeel.h"
41#include "llvm/Transforms/Utils/UnrollLoop.h"
42
43#include <cstdint>
44
45#define DEBUG_TYPE"openmp-ir-builder" "openmp-ir-builder"
46
47using namespace llvm;
48using namespace omp;
49
50static cl::opt<bool>
51 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
52 cl::desc("Use optimistic attributes describing "
53 "'as-if' properties of runtime calls."),
54 cl::init(false));
55
56static cl::opt<double> UnrollThresholdFactor(
57 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
58 cl::desc("Factor for the unroll threshold to account for code "
59 "simplifications still taking place"),
60 cl::init(1.5));
61
62#ifndef NDEBUG
63/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
64/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
65/// an InsertPoint stores the instruction before something is inserted. For
66/// instance, if both point to the same instruction, two IRBuilders alternating
67/// creating instruction will cause the instructions to be interleaved.
68static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
69 IRBuilder<>::InsertPoint IP2) {
70 if (!IP1.isSet() || !IP2.isSet())
71 return false;
72 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
73}
74
75static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
76 // Valid ordered/unordered and base algorithm combinations.
77 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
78 case OMPScheduleType::UnorderedStaticChunked:
79 case OMPScheduleType::UnorderedStatic:
80 case OMPScheduleType::UnorderedDynamicChunked:
81 case OMPScheduleType::UnorderedGuidedChunked:
82 case OMPScheduleType::UnorderedRuntime:
83 case OMPScheduleType::UnorderedAuto:
84 case OMPScheduleType::UnorderedTrapezoidal:
85 case OMPScheduleType::UnorderedGreedy:
86 case OMPScheduleType::UnorderedBalanced:
87 case OMPScheduleType::UnorderedGuidedIterativeChunked:
88 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
89 case OMPScheduleType::UnorderedSteal:
90 case OMPScheduleType::UnorderedStaticBalancedChunked:
91 case OMPScheduleType::UnorderedGuidedSimd:
92 case OMPScheduleType::UnorderedRuntimeSimd:
93 case OMPScheduleType::OrderedStaticChunked:
94 case OMPScheduleType::OrderedStatic:
95 case OMPScheduleType::OrderedDynamicChunked:
96 case OMPScheduleType::OrderedGuidedChunked:
97 case OMPScheduleType::OrderedRuntime:
98 case OMPScheduleType::OrderedAuto:
99 case OMPScheduleType::OrderdTrapezoidal:
100 case OMPScheduleType::NomergeUnorderedStaticChunked:
101 case OMPScheduleType::NomergeUnorderedStatic:
102 case OMPScheduleType::NomergeUnorderedDynamicChunked:
103 case OMPScheduleType::NomergeUnorderedGuidedChunked:
104 case OMPScheduleType::NomergeUnorderedRuntime:
105 case OMPScheduleType::NomergeUnorderedAuto:
106 case OMPScheduleType::NomergeUnorderedTrapezoidal:
107 case OMPScheduleType::NomergeUnorderedGreedy:
108 case OMPScheduleType::NomergeUnorderedBalanced:
109 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
110 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
111 case OMPScheduleType::NomergeUnorderedSteal:
112 case OMPScheduleType::NomergeOrderedStaticChunked:
113 case OMPScheduleType::NomergeOrderedStatic:
114 case OMPScheduleType::NomergeOrderedDynamicChunked:
115 case OMPScheduleType::NomergeOrderedGuidedChunked:
116 case OMPScheduleType::NomergeOrderedRuntime:
117 case OMPScheduleType::NomergeOrderedAuto:
118 case OMPScheduleType::NomergeOrderedTrapezoidal:
119 break;
120 default:
121 return false;
122 }
123
124 // Must not set both monotonicity modifiers at the same time.
125 OMPScheduleType MonotonicityFlags =
126 SchedType & OMPScheduleType::MonotonicityMask;
127 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
128 return false;
129
130 return true;
131}
132#endif
133
134/// Determine which scheduling algorithm to use, determined from schedule clause
135/// arguments.
136static OMPScheduleType
137getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
138 bool HasSimdModifier) {
139 // Currently, the default schedule it static.
140 switch (ClauseKind) {
141 case OMP_SCHEDULE_Default:
142 case OMP_SCHEDULE_Static:
143 return HasChunks ? OMPScheduleType::BaseStaticChunked
144 : OMPScheduleType::BaseStatic;
145 case OMP_SCHEDULE_Dynamic:
146 return OMPScheduleType::BaseDynamicChunked;
147 case OMP_SCHEDULE_Guided:
148 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
149 : OMPScheduleType::BaseGuidedChunked;
150 case OMP_SCHEDULE_Auto:
151 return llvm::omp::OMPScheduleType::BaseAuto;
152 case OMP_SCHEDULE_Runtime:
153 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
154 : OMPScheduleType::BaseRuntime;
155 }
156 llvm_unreachable("unhandled schedule clause argument")::llvm::llvm_unreachable_internal("unhandled schedule clause argument"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 156)
;
157}
158
159/// Adds ordering modifier flags to schedule type.
160static OMPScheduleType
161getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
162 bool HasOrderedClause) {
163 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 165, __extension__
__PRETTY_FUNCTION__))
164 OMPScheduleType::None &&(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 165, __extension__
__PRETTY_FUNCTION__))
165 "Must not have ordering nor monotonicity flags already set")(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 165, __extension__
__PRETTY_FUNCTION__))
;
166
167 OMPScheduleType OrderingModifier = HasOrderedClause
168 ? OMPScheduleType::ModifierOrdered
169 : OMPScheduleType::ModifierUnordered;
170 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
171
172 // Unsupported combinations
173 if (OrderingScheduleType ==
174 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
175 return OMPScheduleType::OrderedGuidedChunked;
176 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
177 OMPScheduleType::ModifierOrdered))
178 return OMPScheduleType::OrderedRuntime;
179
180 return OrderingScheduleType;
181}
182
183/// Adds monotonicity modifier flags to schedule type.
184static OMPScheduleType
185getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
186 bool HasSimdModifier, bool HasMonotonic,
187 bool HasNonmonotonic, bool HasOrderedClause) {
188 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 190, __extension__
__PRETTY_FUNCTION__))
189 OMPScheduleType::None &&(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 190, __extension__
__PRETTY_FUNCTION__))
190 "Must not have monotonicity flags already set")(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 190, __extension__
__PRETTY_FUNCTION__))
;
191 assert((!HasMonotonic || !HasNonmonotonic) &&(static_cast <bool> ((!HasMonotonic || !HasNonmonotonic
) && "Monotonic and Nonmonotonic are contradicting each other"
) ? void (0) : __assert_fail ("(!HasMonotonic || !HasNonmonotonic) && \"Monotonic and Nonmonotonic are contradicting each other\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 192, __extension__
__PRETTY_FUNCTION__))
192 "Monotonic and Nonmonotonic are contradicting each other")(static_cast <bool> ((!HasMonotonic || !HasNonmonotonic
) && "Monotonic and Nonmonotonic are contradicting each other"
) ? void (0) : __assert_fail ("(!HasMonotonic || !HasNonmonotonic) && \"Monotonic and Nonmonotonic are contradicting each other\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 192, __extension__
__PRETTY_FUNCTION__))
;
193
194 if (HasMonotonic) {
195 return ScheduleType | OMPScheduleType::ModifierMonotonic;
196 } else if (HasNonmonotonic) {
197 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
198 } else {
199 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
200 // If the static schedule kind is specified or if the ordered clause is
201 // specified, and if the nonmonotonic modifier is not specified, the
202 // effect is as if the monotonic modifier is specified. Otherwise, unless
203 // the monotonic modifier is specified, the effect is as if the
204 // nonmonotonic modifier is specified.
205 OMPScheduleType BaseScheduleType =
206 ScheduleType & ~OMPScheduleType::ModifierMask;
207 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
208 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
209 HasOrderedClause) {
210 // The monotonic is used by default in openmp runtime library, so no need
211 // to set it.
212 return ScheduleType;
213 } else {
214 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
215 }
216 }
217}
218
219/// Determine the schedule type using schedule and ordering clause arguments.
220static OMPScheduleType
221computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
222 bool HasSimdModifier, bool HasMonotonicModifier,
223 bool HasNonmonotonicModifier, bool HasOrderedClause) {
224 OMPScheduleType BaseSchedule =
225 getOpenMPBaseScheduleType(ClauseKind, HasChunks, HasSimdModifier);
226 OMPScheduleType OrderedSchedule =
227 getOpenMPOrderingScheduleType(BaseSchedule, HasOrderedClause);
228 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
229 OrderedSchedule, HasSimdModifier, HasMonotonicModifier,
230 HasNonmonotonicModifier, HasOrderedClause);
231
232 assert(isValidWorkshareLoopScheduleType(Result))(static_cast <bool> (isValidWorkshareLoopScheduleType(Result
)) ? void (0) : __assert_fail ("isValidWorkshareLoopScheduleType(Result)"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 232, __extension__
__PRETTY_FUNCTION__))
;
233 return Result;
234}
235
236/// Make \p Source branch to \p Target.
237///
238/// Handles two situations:
239/// * \p Source already has an unconditional branch.
240/// * \p Source is a degenerate block (no terminator because the BB is
241/// the current head of the IR construction).
242static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
243 if (Instruction *Term = Source->getTerminator()) {
244 auto *Br = cast<BranchInst>(Term);
245 assert(!Br->isConditional() &&(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 246, __extension__
__PRETTY_FUNCTION__))
246 "BB's terminator must be an unconditional branch (or degenerate)")(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 246, __extension__
__PRETTY_FUNCTION__))
;
247 BasicBlock *Succ = Br->getSuccessor(0);
248 Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
249 Br->setSuccessor(0, Target);
250 return;
251 }
252
253 auto *NewBr = BranchInst::Create(Target, Source);
254 NewBr->setDebugLoc(DL);
255}
256
257void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
258 bool CreateBranch) {
259 assert(New->getFirstInsertionPt() == New->begin() &&(static_cast <bool> (New->getFirstInsertionPt() == New
->begin() && "Target BB must not have PHI nodes") ?
void (0) : __assert_fail ("New->getFirstInsertionPt() == New->begin() && \"Target BB must not have PHI nodes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 260, __extension__
__PRETTY_FUNCTION__))
260 "Target BB must not have PHI nodes")(static_cast <bool> (New->getFirstInsertionPt() == New
->begin() && "Target BB must not have PHI nodes") ?
void (0) : __assert_fail ("New->getFirstInsertionPt() == New->begin() && \"Target BB must not have PHI nodes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 260, __extension__
__PRETTY_FUNCTION__))
;
261
262 // Move instructions to new block.
263 BasicBlock *Old = IP.getBlock();
264 New->getInstList().splice(New->begin(), Old->getInstList(), IP.getPoint(),
265 Old->end());
266
267 if (CreateBranch)
268 BranchInst::Create(New, Old);
269}
270
271void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
272 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
273 BasicBlock *Old = Builder.GetInsertBlock();
274
275 spliceBB(Builder.saveIP(), New, CreateBranch);
276 if (CreateBranch)
277 Builder.SetInsertPoint(Old->getTerminator());
278 else
279 Builder.SetInsertPoint(Old);
280
281 // SetInsertPoint also updates the Builder's debug location, but we want to
282 // keep the one the Builder was configured to use.
283 Builder.SetCurrentDebugLocation(DebugLoc);
284}
285
286BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
287 llvm::Twine Name) {
288 BasicBlock *Old = IP.getBlock();
289 BasicBlock *New = BasicBlock::Create(
290 Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name,
291 Old->getParent(), Old->getNextNode());
292 spliceBB(IP, New, CreateBranch);
293 New->replaceSuccessorsPhiUsesWith(Old, New);
294 return New;
295}
296
297BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
298 llvm::Twine Name) {
299 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
300 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
301 if (CreateBranch)
302 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
303 else
304 Builder.SetInsertPoint(Builder.GetInsertBlock());
305 // SetInsertPoint also updates the Builder's debug location, but we want to
306 // keep the one the Builder was configured to use.
307 Builder.SetCurrentDebugLocation(DebugLoc);
308 return New;
309}
310
311BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
312 llvm::Twine Name) {
313 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
314 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
315 if (CreateBranch)
316 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
317 else
318 Builder.SetInsertPoint(Builder.GetInsertBlock());
319 // SetInsertPoint also updates the Builder's debug location, but we want to
320 // keep the one the Builder was configured to use.
321 Builder.SetCurrentDebugLocation(DebugLoc);
322 return New;
323}
324
325BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
326 llvm::Twine Suffix) {
327 BasicBlock *Old = Builder.GetInsertBlock();
328 return splitBB(Builder, CreateBranch, Old->getName() + Suffix);
329}
330
331void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
332 LLVMContext &Ctx = Fn.getContext();
333
334 // Get the function's current attributes.
335 auto Attrs = Fn.getAttributes();
336 auto FnAttrs = Attrs.getFnAttrs();
337 auto RetAttrs = Attrs.getRetAttrs();
338 SmallVector<AttributeSet, 4> ArgAttrs;
339 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
340 ArgAttrs.emplace_back(Attrs.getParamAttrs(ArgNo));
341
342#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
343#include "llvm/Frontend/OpenMP/OMPKinds.def"
344
345 // Add attributes to the function declaration.
346 switch (FnID) {
347#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
348 case Enum: \
349 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
350 RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
351 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
352 ArgAttrs[ArgNo] = \
353 ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
354 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
355 break;
356#include "llvm/Frontend/OpenMP/OMPKinds.def"
357 default:
358 // Attributes are optional.
359 break;
360 }
361}
362
363FunctionCallee
364OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
365 FunctionType *FnTy = nullptr;
366 Function *Fn = nullptr;
367
368 // Try to find the declation in the module first.
369 switch (FnID) {
370#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
371 case Enum: \
372 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
373 IsVarArg); \
374 Fn = M.getFunction(Str); \
375 break;
376#include "llvm/Frontend/OpenMP/OMPKinds.def"
377 }
378
379 if (!Fn) {
380 // Create a new declaration if we need one.
381 switch (FnID) {
382#define OMP_RTL(Enum, Str, ...) \
383 case Enum: \
384 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
385 break;
386#include "llvm/Frontend/OpenMP/OMPKinds.def"
387 }
388
389 // Add information if the runtime function takes a callback function
390 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
391 if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
392 LLVMContext &Ctx = Fn->getContext();
393 MDBuilder MDB(Ctx);
394 // Annotate the callback behavior of the runtime function:
395 // - The callback callee is argument number 2 (microtask).
396 // - The first two arguments of the callback callee are unknown (-1).
397 // - All variadic arguments to the runtime function are passed to the
398 // callback callee.
399 Fn->addMetadata(
400 LLVMContext::MD_callback,
401 *MDNode::get(Ctx, {MDB.createCallbackEncoding(
402 2, {-1, -1}, /* VarArgsArePassed */ true)}));
403 }
404 }
405
406 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
407 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
408 addAttributes(FnID, *Fn);
409
410 } else {
411 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
412 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
413 }
414
415 assert(Fn && "Failed to create OpenMP runtime function")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 415, __extension__
__PRETTY_FUNCTION__))
;
416
417 // Cast the function to the expected type if necessary
418 Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo());
419 return {FnTy, C};
420}
421
422Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
423 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
424 auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
425 assert(Fn && "Failed to create OpenMP runtime function pointer")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function pointer"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function pointer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 425, __extension__
__PRETTY_FUNCTION__))
;
426 return Fn;
427}
428
429void OpenMPIRBuilder::initialize() { initializeTypes(M); }
430
431void OpenMPIRBuilder::finalize(Function *Fn) {
432 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
433 SmallVector<BasicBlock *, 32> Blocks;
434 SmallVector<OutlineInfo, 16> DeferredOutlines;
435 for (OutlineInfo &OI : OutlineInfos) {
436 // Skip functions that have not finalized yet; may happen with nested
437 // function generation.
438 if (Fn && OI.getFunction() != Fn) {
439 DeferredOutlines.push_back(OI);
440 continue;
441 }
442
443 ParallelRegionBlockSet.clear();
444 Blocks.clear();
445 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
446
447 Function *OuterFn = OI.getFunction();
448 CodeExtractorAnalysisCache CEAC(*OuterFn);
449 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
450 /* AggregateArgs */ true,
451 /* BlockFrequencyInfo */ nullptr,
452 /* BranchProbabilityInfo */ nullptr,
453 /* AssumptionCache */ nullptr,
454 /* AllowVarArgs */ true,
455 /* AllowAlloca */ true,
456 /* AllocaBlock*/ OI.OuterAllocaBB,
457 /* Suffix */ ".omp_par");
458
459 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before outlining: "
<< *OuterFn << "\n"; } } while (false)
;
460 LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
461 << " Exit: " << OI.ExitBB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
;
462 assert(Extractor.isEligible() &&(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 463, __extension__
__PRETTY_FUNCTION__))
463 "Expected OpenMP outlining to be possible!")(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 463, __extension__
__PRETTY_FUNCTION__))
;
464
465 for (auto *V : OI.ExcludeArgsFromAggregate)
466 Extractor.excludeArgFromAggregate(V);
467
468 Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
469
470 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After outlining: "
<< *OuterFn << "\n"; } } while (false)
;
471 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << " Outlined function: "
<< *OutlinedFn << "\n"; } } while (false)
;
472 assert(OutlinedFn->getReturnType()->isVoidTy() &&(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 473, __extension__
__PRETTY_FUNCTION__))
473 "OpenMP outlined functions should not return a value!")(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 473, __extension__
__PRETTY_FUNCTION__))
;
474
475 // For compability with the clang CG we move the outlined function after the
476 // one with the parallel region.
477 OutlinedFn->removeFromParent();
478 M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);
479
480 // Remove the artificial entry introduced by the extractor right away, we
481 // made our own entry block after all.
482 {
483 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
484 assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB)(static_cast <bool> (ArtificialEntry.getUniqueSuccessor
() == OI.EntryBB) ? void (0) : __assert_fail ("ArtificialEntry.getUniqueSuccessor() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 484, __extension__
__PRETTY_FUNCTION__))
;
485 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry)(static_cast <bool> (OI.EntryBB->getUniquePredecessor
() == &ArtificialEntry) ? void (0) : __assert_fail ("OI.EntryBB->getUniquePredecessor() == &ArtificialEntry"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 485, __extension__
__PRETTY_FUNCTION__))
;
486 // Move instructions from the to-be-deleted ArtificialEntry to the entry
487 // basic block of the parallel region. CodeExtractor generates
488 // instructions to unwrap the aggregate argument and may sink
489 // allocas/bitcasts for values that are solely used in the outlined region
490 // and do not escape.
491 assert(!ArtificialEntry.empty() &&(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to add in the outlined region entry")
? void (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to add in the outlined region entry\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 492, __extension__
__PRETTY_FUNCTION__))
492 "Expected instructions to add in the outlined region entry")(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to add in the outlined region entry")
? void (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to add in the outlined region entry\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 492, __extension__
__PRETTY_FUNCTION__))
;
493 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
494 End = ArtificialEntry.rend();
495 It != End;) {
496 Instruction &I = *It;
497 It++;
498
499 if (I.isTerminator())
500 continue;
501
502 I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
503 }
504
505 OI.EntryBB->moveBefore(&ArtificialEntry);
506 ArtificialEntry.eraseFromParent();
507 }
508 assert(&OutlinedFn->getEntryBlock() == OI.EntryBB)(static_cast <bool> (&OutlinedFn->getEntryBlock(
) == OI.EntryBB) ? void (0) : __assert_fail ("&OutlinedFn->getEntryBlock() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 508, __extension__
__PRETTY_FUNCTION__))
;
509 assert(OutlinedFn && OutlinedFn->getNumUses() == 1)(static_cast <bool> (OutlinedFn && OutlinedFn->
getNumUses() == 1) ? void (0) : __assert_fail ("OutlinedFn && OutlinedFn->getNumUses() == 1"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 509, __extension__
__PRETTY_FUNCTION__))
;
510
511 // Run a user callback, e.g. to add attributes.
512 if (OI.PostOutlineCB)
513 OI.PostOutlineCB(*OutlinedFn);
514 }
515
516 // Remove work items that have been completed.
517 OutlineInfos = std::move(DeferredOutlines);
518}
519
520OpenMPIRBuilder::~OpenMPIRBuilder() {
521 assert(OutlineInfos.empty() && "There must be no outstanding outlinings")(static_cast <bool> (OutlineInfos.empty() && "There must be no outstanding outlinings"
) ? void (0) : __assert_fail ("OutlineInfos.empty() && \"There must be no outstanding outlinings\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 521, __extension__
__PRETTY_FUNCTION__))
;
522}
523
524GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
525 IntegerType *I32Ty = Type::getInt32Ty(M.getContext());
526 auto *GV =
527 new GlobalVariable(M, I32Ty,
528 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
529 ConstantInt::get(I32Ty, Value), Name);
530 GV->setVisibility(GlobalValue::HiddenVisibility);
531
532 return GV;
533}
534
535Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
536 uint32_t SrcLocStrSize,
537 IdentFlag LocFlags,
538 unsigned Reserve2Flags) {
539 // Enable "C-mode".
540 LocFlags |= OMP_IDENT_FLAG_KMPC;
541
542 Constant *&Ident =
543 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
544 if (!Ident) {
545 Constant *I32Null = ConstantInt::getNullValue(Int32);
546 Constant *IdentData[] = {I32Null,
547 ConstantInt::get(Int32, uint32_t(LocFlags)),
548 ConstantInt::get(Int32, Reserve2Flags),
549 ConstantInt::get(Int32, SrcLocStrSize), SrcLocStr};
550 Constant *Initializer =
551 ConstantStruct::get(OpenMPIRBuilder::Ident, IdentData);
552
553 // Look for existing encoding of the location + flags, not needed but
554 // minimizes the difference to the existing solution while we transition.
555 for (GlobalVariable &GV : M.getGlobalList())
556 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
557 if (GV.getInitializer() == Initializer)
558 Ident = &GV;
559
560 if (!Ident) {
561 auto *GV = new GlobalVariable(
562 M, OpenMPIRBuilder::Ident,
563 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
564 nullptr, GlobalValue::NotThreadLocal,
565 M.getDataLayout().getDefaultGlobalsAddressSpace());
566 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
567 GV->setAlignment(Align(8));
568 Ident = GV;
569 }
570 }
571
572 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(Ident, IdentPtr);
573}
574
575Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
576 uint32_t &SrcLocStrSize) {
577 SrcLocStrSize = LocStr.size();
578 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
579 if (!SrcLocStr) {
580 Constant *Initializer =
581 ConstantDataArray::getString(M.getContext(), LocStr);
582
583 // Look for existing encoding of the location, not needed but minimizes the
584 // difference to the existing solution while we transition.
585 for (GlobalVariable &GV : M.getGlobalList())
586 if (GV.isConstant() && GV.hasInitializer() &&
587 GV.getInitializer() == Initializer)
588 return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);
589
590 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
591 /* AddressSpace */ 0, &M);
592 }
593 return SrcLocStr;
594}
595
596Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
597 StringRef FileName,
598 unsigned Line, unsigned Column,
599 uint32_t &SrcLocStrSize) {
600 SmallString<128> Buffer;
601 Buffer.push_back(';');
602 Buffer.append(FileName);
603 Buffer.push_back(';');
604 Buffer.append(FunctionName);
605 Buffer.push_back(';');
606 Buffer.append(std::to_string(Line));
607 Buffer.push_back(';');
608 Buffer.append(std::to_string(Column));
609 Buffer.push_back(';');
610 Buffer.push_back(';');
611 return getOrCreateSrcLocStr(Buffer.str(), SrcLocStrSize);
612}
613
614Constant *
615OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
616 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
617 return getOrCreateSrcLocStr(UnknownLoc, SrcLocStrSize);
618}
619
620Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
621 uint32_t &SrcLocStrSize,
622 Function *F) {
623 DILocation *DIL = DL.get();
624 if (!DIL)
625 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
626 StringRef FileName = M.getName();
627 if (DIFile *DIF = DIL->getFile())
628 if (Optional<StringRef> Source = DIF->getSource())
629 FileName = *Source;
630 StringRef Function = DIL->getScope()->getSubprogram()->getName();
631 if (Function.empty() && F)
632 Function = F->getName();
633 return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
634 DIL->getColumn(), SrcLocStrSize);
635}
636
637Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
638 uint32_t &SrcLocStrSize) {
639 return getOrCreateSrcLocStr(Loc.DL, SrcLocStrSize,
640 Loc.IP.getBlock()->getParent());
641}
642
643Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
644 return Builder.CreateCall(
645 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
646 "omp_global_thread_num");
647}
648
649OpenMPIRBuilder::InsertPointTy
650OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
651 bool ForceSimpleCall, bool CheckCancelFlag) {
652 if (!updateToLocation(Loc))
653 return Loc.IP;
654 return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
655}
656
657OpenMPIRBuilder::InsertPointTy
658OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
659 bool ForceSimpleCall, bool CheckCancelFlag) {
660 // Build call __kmpc_cancel_barrier(loc, thread_id) or
661 // __kmpc_barrier(loc, thread_id);
662
663 IdentFlag BarrierLocFlags;
664 switch (Kind) {
665 case OMPD_for:
666 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
667 break;
668 case OMPD_sections:
669 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
670 break;
671 case OMPD_single:
672 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
673 break;
674 case OMPD_barrier:
675 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
676 break;
677 default:
678 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
679 break;
680 }
681
682 uint32_t SrcLocStrSize;
683 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
684 Value *Args[] = {
685 getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
686 getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
687
688 // If we are in a cancellable parallel region, barriers are cancellation
689 // points.
690 // TODO: Check why we would force simple calls or to ignore the cancel flag.
691 bool UseCancelBarrier =
692 !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
693
694 Value *Result =
695 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
696 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
697 : OMPRTL___kmpc_barrier),
698 Args);
699
700 if (UseCancelBarrier && CheckCancelFlag)
701 emitCancelationCheckImpl(Result, OMPD_parallel);
702
703 return Builder.saveIP();
704}
705
706OpenMPIRBuilder::InsertPointTy
707OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
708 Value *IfCondition,
709 omp::Directive CanceledDirective) {
710 if (!updateToLocation(Loc))
711 return Loc.IP;
712
713 // LLVM utilities like blocks with terminators.
714 auto *UI = Builder.CreateUnreachable();
715
716 Instruction *ThenTI = UI, *ElseTI = nullptr;
717 if (IfCondition)
718 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
719 Builder.SetInsertPoint(ThenTI);
720
721 Value *CancelKind = nullptr;
722 switch (CanceledDirective) {
723#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
724 case DirectiveEnum: \
725 CancelKind = Builder.getInt32(Value); \
726 break;
727#include "llvm/Frontend/OpenMP/OMPKinds.def"
728 default:
729 llvm_unreachable("Unknown cancel kind!")::llvm::llvm_unreachable_internal("Unknown cancel kind!", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 729)
;
730 }
731
732 uint32_t SrcLocStrSize;
733 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
734 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
735 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
736 Value *Result = Builder.CreateCall(
737 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
738 auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
739 if (CanceledDirective == OMPD_parallel) {
740 IRBuilder<>::InsertPointGuard IPG(Builder);
741 Builder.restoreIP(IP);
742 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
743 omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
744 /* CheckCancelFlag */ false);
745 }
746 };
747
748 // The actual cancel logic is shared with others, e.g., cancel_barriers.
749 emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);
750
751 // Update the insertion point and remove the terminator we introduced.
752 Builder.SetInsertPoint(UI->getParent());
753 UI->eraseFromParent();
754
755 return Builder.saveIP();
756}
757
758void OpenMPIRBuilder::emitOffloadingEntry(Constant *Addr, StringRef Name,
759 uint64_t Size, int32_t Flags,
760 StringRef SectionName) {
761 Type *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
762 Type *Int32Ty = Type::getInt32Ty(M.getContext());
763 Type *SizeTy = M.getDataLayout().getIntPtrType(M.getContext());
764
765 Constant *AddrName = ConstantDataArray::getString(M.getContext(), Name);
766
767 // Create the constant string used to look up the symbol in the device.
768 auto *Str =
769 new llvm::GlobalVariable(M, AddrName->getType(), /*isConstant=*/true,
770 llvm::GlobalValue::InternalLinkage, AddrName,
771 ".omp_offloading.entry_name");
772 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
773
774 // Construct the offloading entry.
775 Constant *EntryData[] = {
776 ConstantExpr::getPointerBitCastOrAddrSpaceCast(Addr, Int8PtrTy),
777 ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, Int8PtrTy),
778 ConstantInt::get(SizeTy, Size),
779 ConstantInt::get(Int32Ty, Flags),
780 ConstantInt::get(Int32Ty, 0),
781 };
782 Constant *EntryInitializer =
783 ConstantStruct::get(OpenMPIRBuilder::OffloadEntry, EntryData);
784
785 auto *Entry = new GlobalVariable(
786 M, OpenMPIRBuilder::OffloadEntry,
787 /* isConstant = */ true, GlobalValue::WeakAnyLinkage, EntryInitializer,
788 ".omp_offloading.entry." + Name, nullptr, GlobalValue::NotThreadLocal,
789 M.getDataLayout().getDefaultGlobalsAddressSpace());
790
791 // The entry has to be created in the section the linker expects it to be.
792 Entry->setSection(SectionName);
793 Entry->setAlignment(Align(1));
794}
795
796OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
797 const LocationDescription &Loc, Value *&Return, Value *Ident,
798 Value *DeviceID, Value *NumTeams, Value *NumThreads, Value *HostPtr,
799 ArrayRef<Value *> KernelArgs, ArrayRef<Value *> NoWaitArgs) {
800 if (!updateToLocation(Loc))
801 return Loc.IP;
802
803 auto *KernelArgsPtr =
804 Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");
805 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
806 llvm::Value *Arg =
807 Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I);
808 Builder.CreateAlignedStore(
809 KernelArgs[I], Arg,
810 M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType()));
811 }
812
813 bool HasNoWait = !NoWaitArgs.empty();
814 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
815 NumThreads, HostPtr, KernelArgsPtr};
816 if (HasNoWait)
817 OffloadingArgs.append(NoWaitArgs.begin(), NoWaitArgs.end());
818
819 Return = Builder.CreateCall(
820 HasNoWait
821 ? getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel_nowait)
822 : getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel),
823 OffloadingArgs);
824
825 return Builder.saveIP();
826}
827
828void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
829 omp::Directive CanceledDirective,
830 FinalizeCallbackTy ExitCB) {
831 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 832, __extension__
__PRETTY_FUNCTION__))
832 "Unexpected cancellation!")(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 832, __extension__
__PRETTY_FUNCTION__))
;
833
834 // For a cancel barrier we create two new blocks.
835 BasicBlock *BB = Builder.GetInsertBlock();
836 BasicBlock *NonCancellationBlock;
837 if (Builder.GetInsertPoint() == BB->end()) {
838 // TODO: This branch will not be needed once we moved to the
839 // OpenMPIRBuilder codegen completely.
840 NonCancellationBlock = BasicBlock::Create(
841 BB->getContext(), BB->getName() + ".cont", BB->getParent());
842 } else {
843 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
844 BB->getTerminator()->eraseFromParent();
845 Builder.SetInsertPoint(BB);
846 }
847 BasicBlock *CancellationBlock = BasicBlock::Create(
848 BB->getContext(), BB->getName() + ".cncl", BB->getParent());
849
850 // Jump to them based on the return value.
851 Value *Cmp = Builder.CreateIsNull(CancelFlag);
852 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
853 /* TODO weight */ nullptr, nullptr);
854
855 // From the cancellation block we finalize all variables and go to the
856 // post finalization block that is known to the FiniCB callback.
857 Builder.SetInsertPoint(CancellationBlock);
858 if (ExitCB)
859 ExitCB(Builder.saveIP());
860 auto &FI = FinalizationStack.back();
861 FI.FiniCB(Builder.saveIP());
862
863 // The continuation block is where code generation continues.
864 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
865}
866
867IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
868 const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
869 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
870 FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
871 omp::ProcBindKind ProcBind, bool IsCancellable) {
872 assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous")(static_cast <bool> (!isConflictIP(Loc.IP, OuterAllocaIP
) && "IPs must not be ambiguous") ? void (0) : __assert_fail
("!isConflictIP(Loc.IP, OuterAllocaIP) && \"IPs must not be ambiguous\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 872, __extension__
__PRETTY_FUNCTION__))
;
873
874 if (!updateToLocation(Loc))
875 return Loc.IP;
876
877 uint32_t SrcLocStrSize;
878 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
879 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
880 Value *ThreadID = getOrCreateThreadID(Ident);
881
882 if (NumThreads) {
883 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
884 Value *Args[] = {
885 Ident, ThreadID,
886 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
887 Builder.CreateCall(
888 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
889 }
890
891 if (ProcBind != OMP_PROC_BIND_default) {
892 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
893 Value *Args[] = {
894 Ident, ThreadID,
895 ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
896 Builder.CreateCall(
897 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
898 }
899
900 BasicBlock *InsertBB = Builder.GetInsertBlock();
901 Function *OuterFn = InsertBB->getParent();
902
903 // Save the outer alloca block because the insertion iterator may get
904 // invalidated and we still need this later.
905 BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
906
907 // Vector to remember instructions we used only during the modeling but which
908 // we want to delete at the end.
909 SmallVector<Instruction *, 4> ToBeDeleted;
910
911 // Change the location to the outer alloca insertion point to create and
912 // initialize the allocas we pass into the parallel region.
913 Builder.restoreIP(OuterAllocaIP);
914 AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
915 AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
916
917 // If there is an if condition we actually use the TIDAddr and ZeroAddr in the
918 // program, otherwise we only need them for modeling purposes to get the
919 // associated arguments in the outlined function. In the former case,
920 // initialize the allocas properly, in the latter case, delete them later.
921 if (IfCondition) {
922 Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
923 Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
924 } else {
925 ToBeDeleted.push_back(TIDAddr);
926 ToBeDeleted.push_back(ZeroAddr);
927 }
928
929 // Create an artificial insertion point that will also ensure the blocks we
930 // are about to split are not degenerated.
931 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
932
933 Instruction *ThenTI = UI, *ElseTI = nullptr;
934 if (IfCondition)
935 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
936
937 BasicBlock *ThenBB = ThenTI->getParent();
938 BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
939 BasicBlock *PRegBodyBB =
940 PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
941 BasicBlock *PRegPreFiniBB =
942 PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
943 BasicBlock *PRegExitBB =
944 PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");
945
946 auto FiniCBWrapper = [&](InsertPointTy IP) {
947 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
948 // target to the region exit block.
949 if (IP.getBlock()->end() == IP.getPoint()) {
950 IRBuilder<>::InsertPointGuard IPG(Builder);
951 Builder.restoreIP(IP);
952 Instruction *I = Builder.CreateBr(PRegExitBB);
953 IP = InsertPointTy(I->getParent(), I->getIterator());
954 }
955 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 957, __extension__
__PRETTY_FUNCTION__))
956 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 957, __extension__
__PRETTY_FUNCTION__))
957 "Unexpected insertion point for finalization call!")(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 957, __extension__
__PRETTY_FUNCTION__))
;
958 return FiniCB(IP);
959 };
960
961 FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
962
963 // Generate the privatization allocas in the block that will become the entry
964 // of the outlined function.
965 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
966 InsertPointTy InnerAllocaIP = Builder.saveIP();
967
968 AllocaInst *PrivTIDAddr =
969 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
970 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
971
972 // Add some fake uses for OpenMP provided arguments.
973 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
974 Instruction *ZeroAddrUse =
975 Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");
976 ToBeDeleted.push_back(ZeroAddrUse);
977
978 // ThenBB
979 // |
980 // V
981 // PRegionEntryBB <- Privatization allocas are placed here.
982 // |
983 // V
984 // PRegionBodyBB <- BodeGen is invoked here.
985 // |
986 // V
987 // PRegPreFiniBB <- The block we will start finalization from.
988 // |
989 // V
990 // PRegionExitBB <- A common exit to simplify block collection.
991 //
992
993 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
994
995 // Let the caller create the body.
996 assert(BodyGenCB && "Expected body generation callback!")(static_cast <bool> (BodyGenCB && "Expected body generation callback!"
) ? void (0) : __assert_fail ("BodyGenCB && \"Expected body generation callback!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 996, __extension__
__PRETTY_FUNCTION__))
;
997 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
998 BodyGenCB(InnerAllocaIP, CodeGenIP);
999
1000 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
1001
1002 FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
1003 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1004 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1005 llvm::LLVMContext &Ctx = F->getContext();
1006 MDBuilder MDB(Ctx);
1007 // Annotate the callback behavior of the __kmpc_fork_call:
1008 // - The callback callee is argument number 2 (microtask).
1009 // - The first two arguments of the callback callee are unknown (-1).
1010 // - All variadic arguments to the __kmpc_fork_call are passed to the
1011 // callback callee.
1012 F->addMetadata(
1013 llvm::LLVMContext::MD_callback,
1014 *llvm::MDNode::get(
1015 Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
1016 /* VarArgsArePassed */ true)}));
1017 }
1018 }
1019
1020 OutlineInfo OI;
1021 OI.PostOutlineCB = [=](Function &OutlinedFn) {
1022 // Add some known attributes.
1023 OutlinedFn.addParamAttr(0, Attribute::NoAlias);
1024 OutlinedFn.addParamAttr(1, Attribute::NoAlias);
1025 OutlinedFn.addFnAttr(Attribute::NoUnwind);
1026 OutlinedFn.addFnAttr(Attribute::NoRecurse);
1027
1028 assert(OutlinedFn.arg_size() >= 2 &&(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1029, __extension__
__PRETTY_FUNCTION__))
1029 "Expected at least tid and bounded tid as arguments")(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1029, __extension__
__PRETTY_FUNCTION__))
;
1030 unsigned NumCapturedVars =
1031 OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1032
1033 CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
1034 CI->getParent()->setName("omp_parallel");
1035 Builder.SetInsertPoint(CI);
1036
1037 // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
1038 Value *ForkCallArgs[] = {
1039 Ident, Builder.getInt32(NumCapturedVars),
1040 Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
1041
1042 SmallVector<Value *, 16> RealArgs;
1043 RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
1044 RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());
1045
1046 Builder.CreateCall(RTLFn, RealArgs);
1047
1048 LLVM_DEBUG(dbgs() << "With fork_call placed: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
1049 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
1050
1051 InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());
1052
1053 // Initialize the local TID stack location with the argument value.
1054 Builder.SetInsertPoint(PrivTID);
1055 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1056 Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
1057
1058 // If no "if" clause was present we do not need the call created during
1059 // outlining, otherwise we reuse it in the serialized parallel region.
1060 if (!ElseTI) {
1061 CI->eraseFromParent();
1062 } else {
1063
1064 // If an "if" clause was present we are now generating the serialized
1065 // version into the "else" branch.
1066 Builder.SetInsertPoint(ElseTI);
1067
1068 // Build calls __kmpc_serialized_parallel(&Ident, GTid);
1069 Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
1070 Builder.CreateCall(
1071 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
1072 SerializedParallelCallArgs);
1073
1074 // OutlinedFn(&GTid, &zero, CapturedStruct);
1075 CI->removeFromParent();
1076 Builder.Insert(CI);
1077
1078 // __kmpc_end_serialized_parallel(&Ident, GTid);
1079 Value *EndArgs[] = {Ident, ThreadID};
1080 Builder.CreateCall(
1081 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
1082 EndArgs);
1083
1084 LLVM_DEBUG(dbgs() << "With serialized parallel region: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
1085 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
1086 }
1087
1088 for (Instruction *I : ToBeDeleted)
1089 I->eraseFromParent();
1090 };
1091
1092 // Adjust the finalization stack, verify the adjustment, and call the
1093 // finalize function a last time to finalize values between the pre-fini
1094 // block and the exit block if we left the parallel "the normal way".
1095 auto FiniInfo = FinalizationStack.pop_back_val();
1096 (void)FiniInfo;
1097 assert(FiniInfo.DK == OMPD_parallel &&(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1098, __extension__
__PRETTY_FUNCTION__))
1098 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1098, __extension__
__PRETTY_FUNCTION__))
;
1099
1100 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
1101
1102 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
1103 FiniCB(PreFiniIP);
1104
1105 OI.OuterAllocaBB = OuterAllocaBlock;
1106 OI.EntryBB = PRegEntryBB;
1107 OI.ExitBB = PRegExitBB;
1108
1109 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
1110 SmallVector<BasicBlock *, 32> Blocks;
1111 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
1112
1113 // Ensure a single exit node for the outlined region by creating one.
1114 // We might have multiple incoming edges to the exit now due to finalizations,
1115 // e.g., cancel calls that cause the control flow to leave the region.
1116 BasicBlock *PRegOutlinedExitBB = PRegExitBB;
1117 PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
1118 PRegOutlinedExitBB->setName("omp.par.outlined.exit");
1119 Blocks.push_back(PRegOutlinedExitBB);
1120
1121 CodeExtractorAnalysisCache CEAC(*OuterFn);
1122 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
1123 /* AggregateArgs */ false,
1124 /* BlockFrequencyInfo */ nullptr,
1125 /* BranchProbabilityInfo */ nullptr,
1126 /* AssumptionCache */ nullptr,
1127 /* AllowVarArgs */ true,
1128 /* AllowAlloca */ true,
1129 /* AllocationBlock */ OuterAllocaBlock,
1130 /* Suffix */ ".omp_par");
1131
1132 // Find inputs to, outputs from the code region.
1133 BasicBlock *CommonExit = nullptr;
1134 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
1135 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1136 Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);
1137
1138 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before privatization: "
<< *OuterFn << "\n"; } } while (false)
;
1139
1140 FunctionCallee TIDRTLFn =
1141 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
1142
1143 auto PrivHelper = [&](Value &V) {
1144 if (&V == TIDAddr || &V == ZeroAddr) {
1145 OI.ExcludeArgsFromAggregate.push_back(&V);
1146 return;
1147 }
1148
1149 SetVector<Use *> Uses;
1150 for (Use &U : V.uses())
1151 if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
1152 if (ParallelRegionBlockSet.count(UserI->getParent()))
1153 Uses.insert(&U);
1154
1155 // __kmpc_fork_call expects extra arguments as pointers. If the input
1156 // already has a pointer type, everything is fine. Otherwise, store the
1157 // value onto stack and load it back inside the to-be-outlined region. This
1158 // will ensure only the pointer will be passed to the function.
1159 // FIXME: if there are more than 15 trailing arguments, they must be
1160 // additionally packed in a struct.
1161 Value *Inner = &V;
1162 if (!V.getType()->isPointerTy()) {
1163 IRBuilder<>::InsertPointGuard Guard(Builder);
1164 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { llvm::dbgs() << "Forwarding input as pointer: "
<< V << "\n"; } } while (false)
;
1165
1166 Builder.restoreIP(OuterAllocaIP);
1167 Value *Ptr =
1168 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
1169
1170 // Store to stack at end of the block that currently branches to the entry
1171 // block of the to-be-outlined region.
1172 Builder.SetInsertPoint(InsertBB,
1173 InsertBB->getTerminator()->getIterator());
1174 Builder.CreateStore(&V, Ptr);
1175
1176 // Load back next to allocations in the to-be-outlined region.
1177 Builder.restoreIP(InnerAllocaIP);
1178 Inner = Builder.CreateLoad(V.getType(), Ptr);
1179 }
1180
1181 Value *ReplacementValue = nullptr;
1182 CallInst *CI = dyn_cast<CallInst>(&V);
1183 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
1184 ReplacementValue = PrivTID;
1185 } else {
1186 Builder.restoreIP(
1187 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
1188 assert(ReplacementValue &&(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1189, __extension__
__PRETTY_FUNCTION__))
1189 "Expected copy/create callback to set replacement value!")(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1189, __extension__
__PRETTY_FUNCTION__))
;
1190 if (ReplacementValue == &V)
1191 return;
1192 }
1193
1194 for (Use *UPtr : Uses)
1195 UPtr->set(ReplacementValue);
1196 };
1197
1198 // Reset the inner alloca insertion as it will be used for loading the values
1199 // wrapped into pointers before passing them into the to-be-outlined region.
1200 // Configure it to insert immediately after the fake use of zero address so
1201 // that they are available in the generated body and so that the
1202 // OpenMP-related values (thread ID and zero address pointers) remain leading
1203 // in the argument list.
1204 InnerAllocaIP = IRBuilder<>::InsertPoint(
1205 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
1206
1207 // Reset the outer alloca insertion point to the entry of the relevant block
1208 // in case it was invalidated.
1209 OuterAllocaIP = IRBuilder<>::InsertPoint(
1210 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
1211
1212 for (Value *Input : Inputs) {
1213 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Captured input: " <<
*Input << "\n"; } } while (false)
;
1214 PrivHelper(*Input);
1215 }
1216 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1217 for (Value *Output : Outputs)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1218 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1219 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
;
1220 assert(Outputs.empty() &&(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1221, __extension__
__PRETTY_FUNCTION__))
1221 "OpenMP outlining should not produce live-out values!")(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1221, __extension__
__PRETTY_FUNCTION__))
;
1222
1223 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After privatization: "
<< *OuterFn << "\n"; } } while (false)
;
1224 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1225 for (auto *BB : Blocks)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1226 dbgs() << " PBR: " << BB->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1227 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
;
1228
1229 // Register the outlined info.
1230 addOutlineInfo(std::move(OI));
1231
1232 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
1233 UI->eraseFromParent();
1234
1235 return AfterIP;
1236}
1237
1238void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
1239 // Build call void __kmpc_flush(ident_t *loc)
1240 uint32_t SrcLocStrSize;
1241 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1242 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
1243
1244 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
1245}
1246
1247void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
1248 if (!updateToLocation(Loc))
1249 return;
1250 emitFlush(Loc);
1251}
1252
1253void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
1254 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
1255 // global_tid);
1256 uint32_t SrcLocStrSize;
1257 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1258 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1259 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
1260
1261 // Ignore return result until untied tasks are supported.
1262 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
1263 Args);
1264}
1265
1266void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
1267 if (!updateToLocation(Loc))
1268 return;
1269 emitTaskwaitImpl(Loc);
1270}
1271
1272void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
1273 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
1274 uint32_t SrcLocStrSize;
1275 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1276 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1277 Constant *I32Null = ConstantInt::getNullValue(Int32);
1278 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
1279
1280 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
1281 Args);
1282}
1283
1284void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
1285 if (!updateToLocation(Loc))
1286 return;
1287 emitTaskyieldImpl(Loc);
1288}
1289
1290OpenMPIRBuilder::InsertPointTy
1291OpenMPIRBuilder::createTask(const LocationDescription &Loc,
1292 InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
1293 bool Tied, Value *Final, Value *IfCondition) {
1294 if (!updateToLocation(Loc))
1295 return InsertPointTy();
1296
1297 uint32_t SrcLocStrSize;
1298 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1299 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1300 // The current basic block is split into four basic blocks. After outlining,
1301 // they will be mapped as follows:
1302 // ```
1303 // def current_fn() {
1304 // current_basic_block:
1305 // br label %task.exit
1306 // task.exit:
1307 // ; instructions after task
1308 // }
1309 // def outlined_fn() {
1310 // task.alloca:
1311 // br label %task.body
1312 // task.body:
1313 // ret void
1314 // }
1315 // ```
1316 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit");
1317 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body");
1318 BasicBlock *TaskAllocaBB =
1319 splitBB(Builder, /*CreateBranch=*/true, "task.alloca");
1320
1321 OutlineInfo OI;
1322 OI.EntryBB = TaskAllocaBB;
1323 OI.OuterAllocaBB = AllocaIP.getBlock();
1324 OI.ExitBB = TaskExitBB;
1325 OI.PostOutlineCB = [this, Ident, Tied, Final,
1326 IfCondition](Function &OutlinedFn) {
1327 // The input IR here looks like the following-
1328 // ```
1329 // func @current_fn() {
1330 // outlined_fn(%args)
1331 // }
1332 // func @outlined_fn(%args) { ... }
1333 // ```
1334 //
1335 // This is changed to the following-
1336 //
1337 // ```
1338 // func @current_fn() {
1339 // runtime_call(..., wrapper_fn, ...)
1340 // }
1341 // func @wrapper_fn(..., %args) {
1342 // outlined_fn(%args)
1343 // }
1344 // func @outlined_fn(%args) { ... }
1345 // ```
1346
1347 // The stale call instruction will be replaced with a new call instruction
1348 // for runtime call with a wrapper function.
1349 assert(OutlinedFn.getNumUses() == 1 &&(static_cast <bool> (OutlinedFn.getNumUses() == 1 &&
"there must be a single user for the outlined function") ? void
(0) : __assert_fail ("OutlinedFn.getNumUses() == 1 && \"there must be a single user for the outlined function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1350, __extension__
__PRETTY_FUNCTION__))
1350 "there must be a single user for the outlined function")(static_cast <bool> (OutlinedFn.getNumUses() == 1 &&
"there must be a single user for the outlined function") ? void
(0) : __assert_fail ("OutlinedFn.getNumUses() == 1 && \"there must be a single user for the outlined function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1350, __extension__
__PRETTY_FUNCTION__))
;
1351 CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
1352
1353 // HasTaskData is true if any variables are captured in the outlined region,
1354 // false otherwise.
1355 bool HasTaskData = StaleCI->arg_size() > 0;
1356 Builder.SetInsertPoint(StaleCI);
1357
1358 // Gather the arguments for emitting the runtime call for
1359 // @__kmpc_omp_task_alloc
1360 Function *TaskAllocFn =
1361 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
1362
1363 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
1364 // call.
1365 Value *ThreadID = getOrCreateThreadID(Ident);
1366
1367 // Argument - `flags`
1368 // Task is tied iff (Flags & 1) == 1.
1369 // Task is untied iff (Flags & 1) == 0.
1370 // Task is final iff (Flags & 2) == 2.
1371 // Task is not final iff (Flags & 2) == 0.
1372 // TODO: Handle the other flags.
1373 Value *Flags = Builder.getInt32(Tied);
1374 if (Final) {
1375 Value *FinalFlag =
1376 Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0));
1377 Flags = Builder.CreateOr(FinalFlag, Flags);
1378 }
1379
1380 // Argument - `sizeof_kmp_task_t` (TaskSize)
1381 // Tasksize refers to the size in bytes of kmp_task_t data structure
1382 // including private vars accessed in task.
1383 Value *TaskSize = Builder.getInt64(0);
1384 if (HasTaskData) {
1385 AllocaInst *ArgStructAlloca =
1386 dyn_cast<AllocaInst>(StaleCI->getArgOperand(0));
1387 assert(ArgStructAlloca &&(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1389, __extension__
__PRETTY_FUNCTION__))
1388 "Unable to find the alloca instruction corresponding to arguments "(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1389, __extension__
__PRETTY_FUNCTION__))
1389 "for extracted function")(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1389, __extension__
__PRETTY_FUNCTION__))
;
1390 StructType *ArgStructType =
1391 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
1392 assert(ArgStructType && "Unable to find struct type corresponding to "(static_cast <bool> (ArgStructType && "Unable to find struct type corresponding to "
"arguments for extracted function") ? void (0) : __assert_fail
("ArgStructType && \"Unable to find struct type corresponding to \" \"arguments for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1393, __extension__
__PRETTY_FUNCTION__))
1393 "arguments for extracted function")(static_cast <bool> (ArgStructType && "Unable to find struct type corresponding to "
"arguments for extracted function") ? void (0) : __assert_fail
("ArgStructType && \"Unable to find struct type corresponding to \" \"arguments for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1393, __extension__
__PRETTY_FUNCTION__))
;
1394 TaskSize =
1395 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
1396 }
1397
1398 // TODO: Argument - sizeof_shareds
1399
1400 // Argument - task_entry (the wrapper function)
1401 // If the outlined function has some captured variables (i.e. HasTaskData is
1402 // true), then the wrapper function will have an additional argument (the
1403 // struct containing captured variables). Otherwise, no such argument will
1404 // be present.
1405 SmallVector<Type *> WrapperArgTys{Builder.getInt32Ty()};
1406 if (HasTaskData)
1407 WrapperArgTys.push_back(OutlinedFn.getArg(0)->getType());
1408 FunctionCallee WrapperFuncVal = M.getOrInsertFunction(
1409 (Twine(OutlinedFn.getName()) + ".wrapper").str(),
1410 FunctionType::get(Builder.getInt32Ty(), WrapperArgTys, false));
1411 Function *WrapperFunc = dyn_cast<Function>(WrapperFuncVal.getCallee());
1412 PointerType *WrapperFuncBitcastType =
1413 FunctionType::get(Builder.getInt32Ty(),
1414 {Builder.getInt32Ty(), Builder.getInt8PtrTy()}, false)
1415 ->getPointerTo();
1416 Value *WrapperFuncBitcast =
1417 ConstantExpr::getBitCast(WrapperFunc, WrapperFuncBitcastType);
1418
1419 // Emit the @__kmpc_omp_task_alloc runtime call
1420 // The runtime call returns a pointer to an area where the task captured
1421 // variables must be copied before the task is run (NewTaskData)
1422 CallInst *NewTaskData = Builder.CreateCall(
1423 TaskAllocFn,
1424 {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
1425 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/Builder.getInt64(0),
1426 /*task_func=*/WrapperFuncBitcast});
1427
1428 // Copy the arguments for outlined function
1429 if (HasTaskData) {
1430 Value *TaskData = StaleCI->getArgOperand(0);
1431 Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());
1432 Builder.CreateMemCpy(NewTaskData, Alignment, TaskData, Alignment,
1433 TaskSize);
1434 }
1435
1436 // In the presence of the `if` clause, the following IR is generated:
1437 // ...
1438 // %data = call @__kmpc_omp_task_alloc(...)
1439 // br i1 %if_condition, label %then, label %else
1440 // then:
1441 // call @__kmpc_omp_task(...)
1442 // br label %exit
1443 // else:
1444 // call @__kmpc_omp_task_begin_if0(...)
1445 // call @wrapper_fn(...)
1446 // call @__kmpc_omp_task_complete_if0(...)
1447 // br label %exit
1448 // exit:
1449 // ...
1450 if (IfCondition) {
1451 // `SplitBlockAndInsertIfThenElse` requires the block to have a
1452 // terminator.
1453 BasicBlock *NewBasicBlock =
1454 splitBB(Builder, /*CreateBranch=*/true, "if.end");
1455 Instruction *IfTerminator =
1456 NewBasicBlock->getSinglePredecessor()->getTerminator();
1457 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
1458 Builder.SetInsertPoint(IfTerminator);
1459 SplitBlockAndInsertIfThenElse(IfCondition, IfTerminator, &ThenTI,
1460 &ElseTI);
1461 Builder.SetInsertPoint(ElseTI);
1462 Function *TaskBeginFn =
1463 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);
1464 Function *TaskCompleteFn =
1465 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);
1466 Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, NewTaskData});
1467 if (HasTaskData)
1468 Builder.CreateCall(WrapperFunc, {ThreadID, NewTaskData});
1469 else
1470 Builder.CreateCall(WrapperFunc, {ThreadID});
1471 Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, NewTaskData});
1472 Builder.SetInsertPoint(ThenTI);
1473 }
1474 // Emit the @__kmpc_omp_task runtime call to spawn the task
1475 Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);
1476 Builder.CreateCall(TaskFn, {Ident, ThreadID, NewTaskData});
1477
1478 StaleCI->eraseFromParent();
1479
1480 // Emit the body for wrapper function
1481 BasicBlock *WrapperEntryBB =
1482 BasicBlock::Create(M.getContext(), "", WrapperFunc);
1483 Builder.SetInsertPoint(WrapperEntryBB);
1484 if (HasTaskData)
1485 Builder.CreateCall(&OutlinedFn, {WrapperFunc->getArg(1)});
1486 else
1487 Builder.CreateCall(&OutlinedFn);
1488 Builder.CreateRet(Builder.getInt32(0));
1489 };
1490
1491 addOutlineInfo(std::move(OI));
1492
1493 InsertPointTy TaskAllocaIP =
1494 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
1495 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
1496 BodyGenCB(TaskAllocaIP, TaskBodyIP);
1497 Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin());
1498
1499 return Builder.saveIP();
1500}
1501
1502OpenMPIRBuilder::InsertPointTy
1503OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
1504 InsertPointTy AllocaIP,
1505 BodyGenCallbackTy BodyGenCB) {
1506 if (!updateToLocation(Loc))
1507 return InsertPointTy();
1508
1509 uint32_t SrcLocStrSize;
1510 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1511 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1512 Value *ThreadID = getOrCreateThreadID(Ident);
1513
1514 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
1515 Function *TaskgroupFn =
1516 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup);
1517 Builder.CreateCall(TaskgroupFn, {Ident, ThreadID});
1518
1519 BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
1520 BodyGenCB(AllocaIP, Builder.saveIP());
1521
1522 Builder.SetInsertPoint(TaskgroupExitBB);
1523 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
1524 Function *EndTaskgroupFn =
1525 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup);
1526 Builder.CreateCall(EndTaskgroupFn, {Ident, ThreadID});
1527
1528 return Builder.saveIP();
1529}
1530
1531OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
1532 const LocationDescription &Loc, InsertPointTy AllocaIP,
1533 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
1534 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
1535 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required")(static_cast <bool> (!isConflictIP(AllocaIP, Loc.IP) &&
"Dedicated IP allocas required") ? void (0) : __assert_fail (
"!isConflictIP(AllocaIP, Loc.IP) && \"Dedicated IP allocas required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1535, __extension__
__PRETTY_FUNCTION__))
;
1536
1537 if (!updateToLocation(Loc))
1538 return Loc.IP;
1539
1540 auto FiniCBWrapper = [&](InsertPointTy IP) {
1541 if (IP.getBlock()->end() != IP.getPoint())
1542 return FiniCB(IP);
1543 // This must be done otherwise any nested constructs using FinalizeOMPRegion
1544 // will fail because that function requires the Finalization Basic Block to
1545 // have a terminator, which is already removed by EmitOMPRegionBody.
1546 // IP is currently at cancelation block.
1547 // We need to backtrack to the condition block to fetch
1548 // the exit block and create a branch from cancelation
1549 // to exit block.
1550 IRBuilder<>::InsertPointGuard IPG(Builder);
1551 Builder.restoreIP(IP);
1552 auto *CaseBB = IP.getBlock()->getSinglePredecessor();
1553 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1554 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
1555 Instruction *I = Builder.CreateBr(ExitBB);
1556 IP = InsertPointTy(I->getParent(), I->getIterator());
1557 return FiniCB(IP);
1558 };
1559
1560 FinalizationStack.push_back({FiniCBWrapper, OMPD_sections, IsCancellable});
1561
1562 // Each section is emitted as a switch case
1563 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
1564 // -> OMP.createSection() which generates the IR for each section
1565 // Iterate through all sections and emit a switch construct:
1566 // switch (IV) {
1567 // case 0:
1568 // <SectionStmt[0]>;
1569 // break;
1570 // ...
1571 // case <NumSection> - 1:
1572 // <SectionStmt[<NumSection> - 1]>;
1573 // break;
1574 // }
1575 // ...
1576 // section_loop.after:
1577 // <FiniCB>;
1578 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
1579 Builder.restoreIP(CodeGenIP);
1580 BasicBlock *Continue =
1581 splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");
1582 Function *CurFn = Continue->getParent();
1583 SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue);
1584
1585 unsigned CaseNumber = 0;
1586 for (auto SectionCB : SectionCBs) {
1587 BasicBlock *CaseBB = BasicBlock::Create(
1588 M.getContext(), "omp_section_loop.body.case", CurFn, Continue);
1589 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
1590 Builder.SetInsertPoint(CaseBB);
1591 BranchInst *CaseEndBr = Builder.CreateBr(Continue);
1592 SectionCB(InsertPointTy(),
1593 {CaseEndBr->getParent(), CaseEndBr->getIterator()});
1594 CaseNumber++;
1595 }
1596 // remove the existing terminator from body BB since there can be no
1597 // terminators after switch/case
1598 };
1599 // Loop body ends here
1600 // LowerBound, UpperBound, and STride for createCanonicalLoop
1601 Type *I32Ty = Type::getInt32Ty(M.getContext());
1602 Value *LB = ConstantInt::get(I32Ty, 0);
1603 Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());
1604 Value *ST = ConstantInt::get(I32Ty, 1);
1605 llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
1606 Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");
1607 InsertPointTy AfterIP =
1608 applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);
1609
1610 // Apply the finalization callback in LoopAfterBB
1611 auto FiniInfo = FinalizationStack.pop_back_val();
1612 assert(FiniInfo.DK == OMPD_sections &&(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1613, __extension__
__PRETTY_FUNCTION__))
1613 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1613, __extension__
__PRETTY_FUNCTION__))
;
1614 if (FinalizeCallbackTy &CB = FiniInfo.FiniCB) {
1615 Builder.restoreIP(AfterIP);
1616 BasicBlock *FiniBB =
1617 splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini");
1618 CB(Builder.saveIP());
1619 AfterIP = {FiniBB, FiniBB->begin()};
1620 }
1621
1622 return AfterIP;
1623}
1624
1625OpenMPIRBuilder::InsertPointTy
1626OpenMPIRBuilder::createSection(const LocationDescription &Loc,
1627 BodyGenCallbackTy BodyGenCB,
1628 FinalizeCallbackTy FiniCB) {
1629 if (!updateToLocation(Loc))
1630 return Loc.IP;
1631
1632 auto FiniCBWrapper = [&](InsertPointTy IP) {
1633 if (IP.getBlock()->end() != IP.getPoint())
1634 return FiniCB(IP);
1635 // This must be done otherwise any nested constructs using FinalizeOMPRegion
1636 // will fail because that function requires the Finalization Basic Block to
1637 // have a terminator, which is already removed by EmitOMPRegionBody.
1638 // IP is currently at cancelation block.
1639 // We need to backtrack to the condition block to fetch
1640 // the exit block and create a branch from cancelation
1641 // to exit block.
1642 IRBuilder<>::InsertPointGuard IPG(Builder);
1643 Builder.restoreIP(IP);
1644 auto *CaseBB = Loc.IP.getBlock();
1645 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1646 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
1647 Instruction *I = Builder.CreateBr(ExitBB);
1648 IP = InsertPointTy(I->getParent(), I->getIterator());
1649 return FiniCB(IP);
1650 };
1651
1652 Directive OMPD = Directive::OMPD_sections;
1653 // Since we are using Finalization Callback here, HasFinalize
1654 // and IsCancellable have to be true
1655 return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,
1656 /*Conditional*/ false, /*hasFinalize*/ true,
1657 /*IsCancellable*/ true);
1658}
1659
1660/// Create a function with a unique name and a "void (i8*, i8*)" signature in
1661/// the given module and return it.
1662Function *getFreshReductionFunc(Module &M) {
1663 Type *VoidTy = Type::getVoidTy(M.getContext());
1664 Type *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
1665 auto *FuncTy =
1666 FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
1667 return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
1668 M.getDataLayout().getDefaultGlobalsAddressSpace(),
1669 ".omp.reduction.func", &M);
1670}
1671
1672OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
1673 const LocationDescription &Loc, InsertPointTy AllocaIP,
1674 ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait) {
1675 for (const ReductionInfo &RI : ReductionInfos) {
1676 (void)RI;
1677 assert(RI.Variable && "expected non-null variable")(static_cast <bool> (RI.Variable && "expected non-null variable"
) ? void (0) : __assert_fail ("RI.Variable && \"expected non-null variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1677, __extension__
__PRETTY_FUNCTION__))
;
1678 assert(RI.PrivateVariable && "expected non-null private variable")(static_cast <bool> (RI.PrivateVariable && "expected non-null private variable"
) ? void (0) : __assert_fail ("RI.PrivateVariable && \"expected non-null private variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1678, __extension__
__PRETTY_FUNCTION__))
;
1679 assert(RI.ReductionGen && "expected non-null reduction generator callback")(static_cast <bool> (RI.ReductionGen && "expected non-null reduction generator callback"
) ? void (0) : __assert_fail ("RI.ReductionGen && \"expected non-null reduction generator callback\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1679, __extension__
__PRETTY_FUNCTION__))
;
1680 assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1682, __extension__
__PRETTY_FUNCTION__))
1681 "expected variables and their private equivalents to have the same "(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1682, __extension__
__PRETTY_FUNCTION__))
1682 "type")(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1682, __extension__
__PRETTY_FUNCTION__))
;
1683 assert(RI.Variable->getType()->isPointerTy() &&(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1684, __extension__
__PRETTY_FUNCTION__))
1684 "expected variables to be pointers")(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1684, __extension__
__PRETTY_FUNCTION__))
;
1685 }
1686
1687 if (!updateToLocation(Loc))
1688 return InsertPointTy();
1689
1690 BasicBlock *InsertBlock = Loc.IP.getBlock();
1691 BasicBlock *ContinuationBlock =
1692 InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
1693 InsertBlock->getTerminator()->eraseFromParent();
1694
1695 // Create and populate array of type-erased pointers to private reduction
1696 // values.
1697 unsigned NumReductions = ReductionInfos.size();
1698 Type *RedArrayTy = ArrayType::get(Builder.getInt8PtrTy(), NumReductions);
1699 Builder.restoreIP(AllocaIP);
1700 Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");
1701
1702 Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
1703
1704 for (auto En : enumerate(ReductionInfos)) {
1705 unsigned Index = En.index();
1706 const ReductionInfo &RI = En.value();
1707 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
1708 RedArrayTy, RedArray, 0, Index, "red.array.elem." + Twine(Index));
1709 Value *Casted =
1710 Builder.CreateBitCast(RI.PrivateVariable, Builder.getInt8PtrTy(),
1711 "private.red.var." + Twine(Index) + ".casted");
1712 Builder.CreateStore(Casted, RedArrayElemPtr);
1713 }
1714
1715 // Emit a call to the runtime function that orchestrates the reduction.
1716 // Declare the reduction function in the process.
1717 Function *Func = Builder.GetInsertBlock()->getParent();
1718 Module *Module = Func->getParent();
1719 Value *RedArrayPtr =
1720 Builder.CreateBitCast(RedArray, Builder.getInt8PtrTy(), "red.array.ptr");
1721 uint32_t SrcLocStrSize;
1722 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1723 bool CanGenerateAtomic =
1724 llvm::all_of(ReductionInfos, [](const ReductionInfo &RI) {
1725 return RI.AtomicReductionGen;
1726 });
1727 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
1728 CanGenerateAtomic
1729 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
1730 : IdentFlag(0));
1731 Value *ThreadId = getOrCreateThreadID(Ident);
1732 Constant *NumVariables = Builder.getInt32(NumReductions);
1733 const DataLayout &DL = Module->getDataLayout();
1734 unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
1735 Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
1736 Function *ReductionFunc = getFreshReductionFunc(*Module);
1737 Value *Lock = getOMPCriticalRegionLock(".reduction");
1738 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
1739 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
1740 : RuntimeFunction::OMPRTL___kmpc_reduce);
1741 CallInst *ReduceCall =
1742 Builder.CreateCall(ReduceFunc,
1743 {Ident, ThreadId, NumVariables, RedArraySize,
1744 RedArrayPtr, ReductionFunc, Lock},
1745 "reduce");
1746
1747 // Create final reduction entry blocks for the atomic and non-atomic case.
1748 // Emit IR that dispatches control flow to one of the blocks based on the
1749 // reduction supporting the atomic mode.
1750 BasicBlock *NonAtomicRedBlock =
1751 BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func);
1752 BasicBlock *AtomicRedBlock =
1753 BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func);
1754 SwitchInst *Switch =
1755 Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);
1756 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);
1757 Switch->addCase(Builder.getInt32(2), AtomicRedBlock);
1758
1759 // Populate the non-atomic reduction using the elementwise reduction function.
1760 // This loads the elements from the global and private variables and reduces
1761 // them before storing back the result to the global variable.
1762 Builder.SetInsertPoint(NonAtomicRedBlock);
1763 for (auto En : enumerate(ReductionInfos)) {
1764 const ReductionInfo &RI = En.value();
1765 Type *ValueType = RI.ElementType;
1766 Value *RedValue = Builder.CreateLoad(ValueType, RI.Variable,
1767 "red.value." + Twine(En.index()));
1768 Value *PrivateRedValue =
1769 Builder.CreateLoad(ValueType, RI.PrivateVariable,
1770 "red.private.value." + Twine(En.index()));
1771 Value *Reduced;
1772 Builder.restoreIP(
1773 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced));
1774 if (!Builder.GetInsertBlock())
1775 return InsertPointTy();
1776 Builder.CreateStore(Reduced, RI.Variable);
1777 }
1778 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
1779 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
1780 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
1781 Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock});
1782 Builder.CreateBr(ContinuationBlock);
1783
1784 // Populate the atomic reduction using the atomic elementwise reduction
1785 // function. There are no loads/stores here because they will be happening
1786 // inside the atomic elementwise reduction.
1787 Builder.SetInsertPoint(AtomicRedBlock);
1788 if (CanGenerateAtomic) {
1789 for (const ReductionInfo &RI : ReductionInfos) {
1790 Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
1791 RI.Variable, RI.PrivateVariable));
1792 if (!Builder.GetInsertBlock())
1793 return InsertPointTy();
1794 }
1795 Builder.CreateBr(ContinuationBlock);
1796 } else {
1797 Builder.CreateUnreachable();
1798 }
1799
1800 // Populate the outlined reduction function using the elementwise reduction
1801 // function. Partial values are extracted from the type-erased array of
1802 // pointers to private variables.
1803 BasicBlock *ReductionFuncBlock =
1804 BasicBlock::Create(Module->getContext(), "", ReductionFunc);
1805 Builder.SetInsertPoint(ReductionFuncBlock);
1806 Value *LHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(0),
1807 RedArrayTy->getPointerTo());
1808 Value *RHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(1),
1809 RedArrayTy->getPointerTo());
1810 for (auto En : enumerate(ReductionInfos)) {
1811 const ReductionInfo &RI = En.value();
1812 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1813 RedArrayTy, LHSArrayPtr, 0, En.index());
1814 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), LHSI8PtrPtr);
1815 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
1816 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
1817 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1818 RedArrayTy, RHSArrayPtr, 0, En.index());
1819 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), RHSI8PtrPtr);
1820 Value *RHSPtr =
1821 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
1822 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
1823 Value *Reduced;
1824 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
1825 if (!Builder.GetInsertBlock())
1826 return InsertPointTy();
1827 Builder.CreateStore(Reduced, LHSPtr);
1828 }
1829 Builder.CreateRetVoid();
1830
1831 Builder.SetInsertPoint(ContinuationBlock);
1832 return Builder.saveIP();
1833}
1834
1835OpenMPIRBuilder::InsertPointTy
1836OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
1837 BodyGenCallbackTy BodyGenCB,
1838 FinalizeCallbackTy FiniCB) {
1839
1840 if (!updateToLocation(Loc))
1841 return Loc.IP;
1842
1843 Directive OMPD = Directive::OMPD_master;
1844 uint32_t SrcLocStrSize;
1845 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1846 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1847 Value *ThreadId = getOrCreateThreadID(Ident);
1848 Value *Args[] = {Ident, ThreadId};
1849
1850 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
1851 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1852
1853 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
1854 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1855
1856 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1857 /*Conditional*/ true, /*hasFinalize*/ true);
1858}
1859
1860OpenMPIRBuilder::InsertPointTy
1861OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
1862 BodyGenCallbackTy BodyGenCB,
1863 FinalizeCallbackTy FiniCB, Value *Filter) {
1864 if (!updateToLocation(Loc))
1865 return Loc.IP;
1866
1867 Directive OMPD = Directive::OMPD_masked;
1868 uint32_t SrcLocStrSize;
1869 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1870 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1871 Value *ThreadId = getOrCreateThreadID(Ident);
1872 Value *Args[] = {Ident, ThreadId, Filter};
1873 Value *ArgsEnd[] = {Ident, ThreadId};
1874
1875 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_masked);
1876 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1877
1878 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_masked);
1879 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd);
1880
1881 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1882 /*Conditional*/ true, /*hasFinalize*/ true);
1883}
1884
1885CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
1886 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
1887 BasicBlock *PostInsertBefore, const Twine &Name) {
1888 Module *M = F->getParent();
1889 LLVMContext &Ctx = M->getContext();
1890 Type *IndVarTy = TripCount->getType();
1891
1892 // Create the basic block structure.
1893 BasicBlock *Preheader =
1894 BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
1895 BasicBlock *Header =
1896 BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
1897 BasicBlock *Cond =
1898 BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
1899 BasicBlock *Body =
1900 BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
1901 BasicBlock *Latch =
1902 BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
1903 BasicBlock *Exit =
1904 BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
1905 BasicBlock *After =
1906 BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
1907
1908 // Use specified DebugLoc for new instructions.
1909 Builder.SetCurrentDebugLocation(DL);
1910
1911 Builder.SetInsertPoint(Preheader);
1912 Builder.CreateBr(Header);
1913
1914 Builder.SetInsertPoint(Header);
1915 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
1916 IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
1917 Builder.CreateBr(Cond);
1918
1919 Builder.SetInsertPoint(Cond);
1920 Value *Cmp =
1921 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
1922 Builder.CreateCondBr(Cmp, Body, Exit);
1923
1924 Builder.SetInsertPoint(Body);
1925 Builder.CreateBr(Latch);
1926
1927 Builder.SetInsertPoint(Latch);
1928 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
1929 "omp_" + Name + ".next", /*HasNUW=*/true);
1930 Builder.CreateBr(Header);
1931 IndVarPHI->addIncoming(Next, Latch);
1932
1933 Builder.SetInsertPoint(Exit);
1934 Builder.CreateBr(After);
1935
1936 // Remember and return the canonical control flow.
1937 LoopInfos.emplace_front();
1938 CanonicalLoopInfo *CL = &LoopInfos.front();
1939
1940 CL->Header = Header;
1941 CL->Cond = Cond;
1942 CL->Latch = Latch;
1943 CL->Exit = Exit;
1944
1945#ifndef NDEBUG
1946 CL->assertOK();
1947#endif
1948 return CL;
1949}
1950
1951CanonicalLoopInfo *
1952OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
1953 LoopBodyGenCallbackTy BodyGenCB,
1954 Value *TripCount, const Twine &Name) {
1955 BasicBlock *BB = Loc.IP.getBlock();
1956 BasicBlock *NextBB = BB->getNextNode();
1957
1958 CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
1959 NextBB, NextBB, Name);
1960 BasicBlock *After = CL->getAfter();
1961
1962 // If location is not set, don't connect the loop.
1963 if (updateToLocation(Loc)) {
1964 // Split the loop at the insertion point: Branch to the preheader and move
1965 // every following instruction to after the loop (the After BB). Also, the
1966 // new successor is the loop's after block.
1967 spliceBB(Builder, After, /*CreateBranch=*/false);
1968 Builder.CreateBr(CL->getPreheader());
1969 }
1970
1971 // Emit the body content. We do it after connecting the loop to the CFG to
1972 // avoid that the callback encounters degenerate BBs.
1973 BodyGenCB(CL->getBodyIP(), CL->getIndVar());
1974
1975#ifndef NDEBUG
1976 CL->assertOK();
1977#endif
1978 return CL;
1979}
1980
1981CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
1982 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
1983 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
1984 InsertPointTy ComputeIP, const Twine &Name) {
1985
1986 // Consider the following difficulties (assuming 8-bit signed integers):
1987 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
1988 // DO I = 1, 100, 50
1989 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
1990 // DO I = 100, 0, -128
1991
1992 // Start, Stop and Step must be of the same integer type.
1993 auto *IndVarTy = cast<IntegerType>(Start->getType());
1994 assert(IndVarTy == Stop->getType() && "Stop type mismatch")(static_cast <bool> (IndVarTy == Stop->getType() &&
"Stop type mismatch") ? void (0) : __assert_fail ("IndVarTy == Stop->getType() && \"Stop type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1994, __extension__
__PRETTY_FUNCTION__))
;
1995 assert(IndVarTy == Step->getType() && "Step type mismatch")(static_cast <bool> (IndVarTy == Step->getType() &&
"Step type mismatch") ? void (0) : __assert_fail ("IndVarTy == Step->getType() && \"Step type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1995, __extension__
__PRETTY_FUNCTION__))
;
1996
1997 LocationDescription ComputeLoc =
1998 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
1999 updateToLocation(ComputeLoc);
2000
2001 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
2002 ConstantInt *One = ConstantInt::get(IndVarTy, 1);
2003
2004 // Like Step, but always positive.
2005 Value *Incr = Step;
2006
2007 // Distance between Start and Stop; always positive.
2008 Value *Span;
2009
2010 // Condition whether there are no iterations are executed at all, e.g. because
2011 // UB < LB.
2012 Value *ZeroCmp;
2013
2014 if (IsSigned) {
2015 // Ensure that increment is positive. If not, negate and invert LB and UB.
2016 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
2017 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
2018 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
2019 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
2020 Span = Builder.CreateSub(UB, LB, "", false, true);
2021 ZeroCmp = Builder.CreateICmp(
2022 InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
2023 } else {
2024 Span = Builder.CreateSub(Stop, Start, "", true);
2025 ZeroCmp = Builder.CreateICmp(
2026 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
2027 }
2028
2029 Value *CountIfLooping;
2030 if (InclusiveStop) {
2031 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
2032 } else {
2033 // Avoid incrementing past stop since it could overflow.
2034 Value *CountIfTwo = Builder.CreateAdd(
2035 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
2036 Value *OneCmp = Builder.CreateICmp(
2037 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
2038 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
2039 }
2040 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
2041 "omp_" + Name + ".tripcount");
2042
2043 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
2044 Builder.restoreIP(CodeGenIP);
2045 Value *Span = Builder.CreateMul(IV, Step);
2046 Value *IndVar = Builder.CreateAdd(Span, Start);
2047 BodyGenCB(Builder.saveIP(), IndVar);
2048 };
2049 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
2050 return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
2051}
2052
2053// Returns an LLVM function to call for initializing loop bounds using OpenMP
2054// static scheduling depending on `type`. Only i32 and i64 are supported by the
2055// runtime. Always interpret integers as unsigned similarly to
2056// CanonicalLoopInfo.
2057static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
2058 OpenMPIRBuilder &OMPBuilder) {
2059 unsigned Bitwidth = Ty->getIntegerBitWidth();
2060 if (Bitwidth == 32)
2061 return OMPBuilder.getOrCreateRuntimeFunction(
2062 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
2063 if (Bitwidth == 64)
2064 return OMPBuilder.getOrCreateRuntimeFunction(
2065 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
2066 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2066)
;
2067}
2068
2069OpenMPIRBuilder::InsertPointTy
2070OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
2071 InsertPointTy AllocaIP,
2072 bool NeedsBarrier) {
2073 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2073, __extension__
__PRETTY_FUNCTION__))
;
2074 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2075, __extension__
__PRETTY_FUNCTION__))
2075 "Require dedicated allocate IP")(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2075, __extension__
__PRETTY_FUNCTION__))
;
2076
2077 // Set up the source location value for OpenMP runtime.
2078 Builder.restoreIP(CLI->getPreheaderIP());
2079 Builder.SetCurrentDebugLocation(DL);
2080
2081 uint32_t SrcLocStrSize;
2082 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2083 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2084
2085 // Declare useful OpenMP runtime functions.
2086 Value *IV = CLI->getIndVar();
2087 Type *IVTy = IV->getType();
2088 FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
2089 FunctionCallee StaticFini =
2090 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
2091
2092 // Allocate space for computed loop bounds as expected by the "init" function.
2093 Builder.restoreIP(AllocaIP);
2094 Type *I32Type = Type::getInt32Ty(M.getContext());
2095 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2096 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
2097 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
2098 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
2099
2100 // At the end of the preheader, prepare for calling the "init" function by
2101 // storing the current loop bounds into the allocated space. A canonical loop
2102 // always iterates from 0 to trip-count with step 1. Note that "init" expects
2103 // and produces an inclusive upper bound.
2104 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2105 Constant *Zero = ConstantInt::get(IVTy, 0);
2106 Constant *One = ConstantInt::get(IVTy, 1);
2107 Builder.CreateStore(Zero, PLowerBound);
2108 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
2109 Builder.CreateStore(UpperBound, PUpperBound);
2110 Builder.CreateStore(One, PStride);
2111
2112 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2113
2114 Constant *SchedulingType = ConstantInt::get(
2115 I32Type, static_cast<int>(OMPScheduleType::UnorderedStatic));
2116
2117 // Call the "init" function and update the trip count of the loop with the
2118 // value it produced.
2119 Builder.CreateCall(StaticInit,
2120 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
2121 PUpperBound, PStride, One, Zero});
2122 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
2123 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
2124 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
2125 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
2126 CLI->setTripCount(TripCount);
2127
2128 // Update all uses of the induction variable except the one in the condition
2129 // block that compares it with the actual upper bound, and the increment in
2130 // the latch block.
2131
2132 CLI->mapIndVar([&](Instruction *OldIV) -> Value * {
2133 Builder.SetInsertPoint(CLI->getBody(),
2134 CLI->getBody()->getFirstInsertionPt());
2135 Builder.SetCurrentDebugLocation(DL);
2136 return Builder.CreateAdd(OldIV, LowerBound);
2137 });
2138
2139 // In the "exit" block, call the "fini" function.
2140 Builder.SetInsertPoint(CLI->getExit(),
2141 CLI->getExit()->getTerminator()->getIterator());
2142 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
2143
2144 // Add the barrier if requested.
2145 if (NeedsBarrier)
2146 createBarrier(LocationDescription(Builder.saveIP(), DL),
2147 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
2148 /* CheckCancelFlag */ false);
2149
2150 InsertPointTy AfterIP = CLI->getAfterIP();
2151 CLI->invalidate();
2152
2153 return AfterIP;
2154}
2155
2156OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
2157 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2158 bool NeedsBarrier, Value *ChunkSize) {
2159 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2159, __extension__
__PRETTY_FUNCTION__))
;
2160 assert(ChunkSize && "Chunk size is required")(static_cast <bool> (ChunkSize && "Chunk size is required"
) ? void (0) : __assert_fail ("ChunkSize && \"Chunk size is required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2160, __extension__
__PRETTY_FUNCTION__))
;
2161
2162 LLVMContext &Ctx = CLI->getFunction()->getContext();
2163 Value *IV = CLI->getIndVar();
2164 Value *OrigTripCount = CLI->getTripCount();
2165 Type *IVTy = IV->getType();
2166 assert(IVTy->getIntegerBitWidth() <= 64 &&(static_cast <bool> (IVTy->getIntegerBitWidth() <=
64 && "Max supported tripcount bitwidth is 64 bits")
? void (0) : __assert_fail ("IVTy->getIntegerBitWidth() <= 64 && \"Max supported tripcount bitwidth is 64 bits\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2167, __extension__
__PRETTY_FUNCTION__))
2167 "Max supported tripcount bitwidth is 64 bits")(static_cast <bool> (IVTy->getIntegerBitWidth() <=
64 && "Max supported tripcount bitwidth is 64 bits")
? void (0) : __assert_fail ("IVTy->getIntegerBitWidth() <= 64 && \"Max supported tripcount bitwidth is 64 bits\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2167, __extension__
__PRETTY_FUNCTION__))
;
2168 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(Ctx)
2169 : Type::getInt64Ty(Ctx);
2170 Type *I32Type = Type::getInt32Ty(M.getContext());
2171 Constant *Zero = ConstantInt::get(InternalIVTy, 0);
2172 Constant *One = ConstantInt::get(InternalIVTy, 1);
2173
2174 // Declare useful OpenMP runtime functions.
2175 FunctionCallee StaticInit =
2176 getKmpcForStaticInitForType(InternalIVTy, M, *this);
2177 FunctionCallee StaticFini =
2178 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
2179
2180 // Allocate space for computed loop bounds as expected by the "init" function.
2181 Builder.restoreIP(AllocaIP);
2182 Builder.SetCurrentDebugLocation(DL);
2183 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2184 Value *PLowerBound =
2185 Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound");
2186 Value *PUpperBound =
2187 Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound");
2188 Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride");
2189
2190 // Set up the source location value for the OpenMP runtime.
2191 Builder.restoreIP(CLI->getPreheaderIP());
2192 Builder.SetCurrentDebugLocation(DL);
2193
2194 // TODO: Detect overflow in ubsan or max-out with current tripcount.
2195 Value *CastedChunkSize =
2196 Builder.CreateZExtOrTrunc(ChunkSize, InternalIVTy, "chunksize");
2197 Value *CastedTripCount =
2198 Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount");
2199
2200 Constant *SchedulingType = ConstantInt::get(
2201 I32Type, static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
2202 Builder.CreateStore(Zero, PLowerBound);
2203 Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One);
2204 Builder.CreateStore(OrigUpperBound, PUpperBound);
2205 Builder.CreateStore(One, PStride);
2206
2207 // Call the "init" function and update the trip count of the loop with the
2208 // value it produced.
2209 uint32_t SrcLocStrSize;
2210 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2211 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2212 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2213 Builder.CreateCall(StaticInit,
2214 {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
2215 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
2216 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
2217 /*pstride=*/PStride, /*incr=*/One,
2218 /*chunk=*/CastedChunkSize});
2219
2220 // Load values written by the "init" function.
2221 Value *FirstChunkStart =
2222 Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb");
2223 Value *FirstChunkStop =
2224 Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub");
2225 Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One);
2226 Value *ChunkRange =
2227 Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range");
2228 Value *NextChunkStride =
2229 Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride");
2230
2231 // Create outer "dispatch" loop for enumerating the chunks.
2232 BasicBlock *DispatchEnter = splitBB(Builder, true);
2233 Value *DispatchCounter;
2234 CanonicalLoopInfo *DispatchCLI = createCanonicalLoop(
2235 {Builder.saveIP(), DL},
2236 [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; },
2237 FirstChunkStart, CastedTripCount, NextChunkStride,
2238 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
2239 "dispatch");
2240
2241 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
2242 // not have to preserve the canonical invariant.
2243 BasicBlock *DispatchBody = DispatchCLI->getBody();
2244 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
2245 BasicBlock *DispatchExit = DispatchCLI->getExit();
2246 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
2247 DispatchCLI->invalidate();
2248
2249 // Rewire the original loop to become the chunk loop inside the dispatch loop.
2250 redirectTo(DispatchAfter, CLI->getAfter(), DL);
2251 redirectTo(CLI->getExit(), DispatchLatch, DL);
2252 redirectTo(DispatchBody, DispatchEnter, DL);
2253
2254 // Prepare the prolog of the chunk loop.
2255 Builder.restoreIP(CLI->getPreheaderIP());
2256 Builder.SetCurrentDebugLocation(DL);
2257
2258 // Compute the number of iterations of the chunk loop.
2259 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2260 Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange);
2261 Value *IsLastChunk =
2262 Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last");
2263 Value *CountUntilOrigTripCount =
2264 Builder.CreateSub(CastedTripCount, DispatchCounter);
2265 Value *ChunkTripCount = Builder.CreateSelect(
2266 IsLastChunk, CountUntilOrigTripCount, ChunkRange, "omp_chunk.tripcount");
2267 Value *BackcastedChunkTC =
2268 Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc");
2269 CLI->setTripCount(BackcastedChunkTC);
2270
2271 // Update all uses of the induction variable except the one in the condition
2272 // block that compares it with the actual upper bound, and the increment in
2273 // the latch block.
2274 Value *BackcastedDispatchCounter =
2275 Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc");
2276 CLI->mapIndVar([&](Instruction *) -> Value * {
2277 Builder.restoreIP(CLI->getBodyIP());
2278 return Builder.CreateAdd(IV, BackcastedDispatchCounter);
2279 });
2280
2281 // In the "exit" block, call the "fini" function.
2282 Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt());
2283 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
2284
2285 // Add the barrier if requested.
2286 if (NeedsBarrier)
2287 createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
2288 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
2289
2290#ifndef NDEBUG
2291 // Even though we currently do not support applying additional methods to it,
2292 // the chunk loop should remain a canonical loop.
2293 CLI->assertOK();
2294#endif
2295
2296 return {DispatchAfter, DispatchAfter->getFirstInsertionPt()};
2297}
2298
2299OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop(
2300 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2301 bool NeedsBarrier, llvm::omp::ScheduleKind SchedKind,
2302 llvm::Value *ChunkSize, bool HasSimdModifier, bool HasMonotonicModifier,
2303 bool HasNonmonotonicModifier, bool HasOrderedClause) {
2304 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
2305 SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
2306 HasNonmonotonicModifier, HasOrderedClause);
2307
2308 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
2309 OMPScheduleType::ModifierOrdered;
2310 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
2311 case OMPScheduleType::BaseStatic:
2312 assert(!ChunkSize && "No chunk size with static-chunked schedule")(static_cast <bool> (!ChunkSize && "No chunk size with static-chunked schedule"
) ? void (0) : __assert_fail ("!ChunkSize && \"No chunk size with static-chunked schedule\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2312, __extension__
__PRETTY_FUNCTION__))
;
2313 if (IsOrdered)
2314 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2315 NeedsBarrier, ChunkSize);
2316 // FIXME: Monotonicity ignored?
2317 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier);
2318
2319 case OMPScheduleType::BaseStaticChunked:
2320 if (IsOrdered)
2321 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2322 NeedsBarrier, ChunkSize);
2323 // FIXME: Monotonicity ignored?
2324 return applyStaticChunkedWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier,
2325 ChunkSize);
2326
2327 case OMPScheduleType::BaseRuntime:
2328 case OMPScheduleType::BaseAuto:
2329 case OMPScheduleType::BaseGreedy:
2330 case OMPScheduleType::BaseBalanced:
2331 case OMPScheduleType::BaseSteal:
2332 case OMPScheduleType::BaseGuidedSimd:
2333 case OMPScheduleType::BaseRuntimeSimd:
2334 assert(!ChunkSize &&(static_cast <bool> (!ChunkSize && "schedule type does not support user-defined chunk sizes"
) ? void (0) : __assert_fail ("!ChunkSize && \"schedule type does not support user-defined chunk sizes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2335, __extension__
__PRETTY_FUNCTION__))
2335 "schedule type does not support user-defined chunk sizes")(static_cast <bool> (!ChunkSize && "schedule type does not support user-defined chunk sizes"
) ? void (0) : __assert_fail ("!ChunkSize && \"schedule type does not support user-defined chunk sizes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2335, __extension__
__PRETTY_FUNCTION__))
;
2336 [[fallthrough]];
2337 case OMPScheduleType::BaseDynamicChunked:
2338 case OMPScheduleType::BaseGuidedChunked:
2339 case OMPScheduleType::BaseGuidedIterativeChunked:
2340 case OMPScheduleType::BaseGuidedAnalyticalChunked:
2341 case OMPScheduleType::BaseStaticBalancedChunked:
2342 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2343 NeedsBarrier, ChunkSize);
2344
2345 default:
2346 llvm_unreachable("Unknown/unimplemented schedule kind")::llvm::llvm_unreachable_internal("Unknown/unimplemented schedule kind"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2346)
;
2347 }
2348}
2349
2350/// Returns an LLVM function to call for initializing loop bounds using OpenMP
2351/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2352/// the runtime. Always interpret integers as unsigned similarly to
2353/// CanonicalLoopInfo.
2354static FunctionCallee
2355getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2356 unsigned Bitwidth = Ty->getIntegerBitWidth();
2357 if (Bitwidth == 32)
2358 return OMPBuilder.getOrCreateRuntimeFunction(
2359 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
2360 if (Bitwidth == 64)
2361 return OMPBuilder.getOrCreateRuntimeFunction(
2362 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
2363 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2363)
;
2364}
2365
2366/// Returns an LLVM function to call for updating the next loop using OpenMP
2367/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2368/// the runtime. Always interpret integers as unsigned similarly to
2369/// CanonicalLoopInfo.
2370static FunctionCallee
2371getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2372 unsigned Bitwidth = Ty->getIntegerBitWidth();
2373 if (Bitwidth == 32)
2374 return OMPBuilder.getOrCreateRuntimeFunction(
2375 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
2376 if (Bitwidth == 64)
2377 return OMPBuilder.getOrCreateRuntimeFunction(
2378 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
2379 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2379)
;
2380}
2381
2382/// Returns an LLVM function to call for finalizing the dynamic loop using
2383/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
2384/// interpret integers as unsigned similarly to CanonicalLoopInfo.
2385static FunctionCallee
2386getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2387 unsigned Bitwidth = Ty->getIntegerBitWidth();
2388 if (Bitwidth == 32)
2389 return OMPBuilder.getOrCreateRuntimeFunction(
2390 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
2391 if (Bitwidth == 64)
2392 return OMPBuilder.getOrCreateRuntimeFunction(
2393 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
2394 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2394)
;
2395}
2396
2397OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
2398 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2399 OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
2400 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2400, __extension__
__PRETTY_FUNCTION__))
;
2401 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2402, __extension__
__PRETTY_FUNCTION__))
2402 "Require dedicated allocate IP")(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2402, __extension__
__PRETTY_FUNCTION__))
;
2403 assert(isValidWorkshareLoopScheduleType(SchedType) &&(static_cast <bool> (isValidWorkshareLoopScheduleType(SchedType
) && "Require valid schedule type") ? void (0) : __assert_fail
("isValidWorkshareLoopScheduleType(SchedType) && \"Require valid schedule type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2404, __extension__
__PRETTY_FUNCTION__))
2404 "Require valid schedule type")(static_cast <bool> (isValidWorkshareLoopScheduleType(SchedType
) && "Require valid schedule type") ? void (0) : __assert_fail
("isValidWorkshareLoopScheduleType(SchedType) && \"Require valid schedule type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2404, __extension__
__PRETTY_FUNCTION__))
;
2405
2406 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
2407 OMPScheduleType::ModifierOrdered;
2408
2409 // Set up the source location value for OpenMP runtime.
2410 Builder.SetCurrentDebugLocation(DL);
2411
2412 uint32_t SrcLocStrSize;
2413 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2414 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2415
2416 // Declare useful OpenMP runtime functions.
2417 Value *IV = CLI->getIndVar();
2418 Type *IVTy = IV->getType();
2419 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(IVTy, M, *this);
2420 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(IVTy, M, *this);
2421
2422 // Allocate space for computed loop bounds as expected by the "init" function.
2423 Builder.restoreIP(AllocaIP);
2424 Type *I32Type = Type::getInt32Ty(M.getContext());
2425 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2426 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
2427 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
2428 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
2429
2430 // At the end of the preheader, prepare for calling the "init" function by
2431 // storing the current loop bounds into the allocated space. A canonical loop
2432 // always iterates from 0 to trip-count with step 1. Note that "init" expects
2433 // and produces an inclusive upper bound.
2434 BasicBlock *PreHeader = CLI->getPreheader();
2435 Builder.SetInsertPoint(PreHeader->getTerminator());
2436 Constant *One = ConstantInt::get(IVTy, 1);
2437 Builder.CreateStore(One, PLowerBound);
2438 Value *UpperBound = CLI->getTripCount();
2439 Builder.CreateStore(UpperBound, PUpperBound);
2440 Builder.CreateStore(One, PStride);
2441
2442 BasicBlock *Header = CLI->getHeader();
2443 BasicBlock *Exit = CLI->getExit();
2444 BasicBlock *Cond = CLI->getCond();
2445 BasicBlock *Latch = CLI->getLatch();
2446 InsertPointTy AfterIP = CLI->getAfterIP();
2447
2448 // The CLI will be "broken" in the code below, as the loop is no longer
2449 // a valid canonical loop.
2450
2451 if (!Chunk)
2452 Chunk = One;
2453
2454 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2455
2456 Constant *SchedulingType =
2457 ConstantInt::get(I32Type, static_cast<int>(SchedType));
2458
2459 // Call the "init" function.
2460 Builder.CreateCall(DynamicInit,
2461 {SrcLoc, ThreadNum, SchedulingType, /* LowerBound */ One,
2462 UpperBound, /* step */ One, Chunk});
2463
2464 // An outer loop around the existing one.
2465 BasicBlock *OuterCond = BasicBlock::Create(
2466 PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond",
2467 PreHeader->getParent());
2468 // This needs to be 32-bit always, so can't use the IVTy Zero above.
2469 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());
2470 Value *Res =
2471 Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter,
2472 PLowerBound, PUpperBound, PStride});
2473 Constant *Zero32 = ConstantInt::get(I32Type, 0);
2474 Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);
2475 Value *LowerBound =
2476 Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");
2477 Builder.CreateCondBr(MoreWork, Header, Exit);
2478
2479 // Change PHI-node in loop header to use outer cond rather than preheader,
2480 // and set IV to the LowerBound.
2481 Instruction *Phi = &Header->front();
2482 auto *PI = cast<PHINode>(Phi);
2483 PI->setIncomingBlock(0, OuterCond);
2484 PI->setIncomingValue(0, LowerBound);
2485
2486 // Then set the pre-header to jump to the OuterCond
2487 Instruction *Term = PreHeader->getTerminator();
2488 auto *Br = cast<BranchInst>(Term);
2489 Br->setSuccessor(0, OuterCond);
2490
2491 // Modify the inner condition:
2492 // * Use the UpperBound returned from the DynamicNext call.
2493 // * jump to the loop outer loop when done with one of the inner loops.
2494 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());
2495 UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");
2496 Instruction *Comp = &*Builder.GetInsertPoint();
2497 auto *CI = cast<CmpInst>(Comp);
2498 CI->setOperand(1, UpperBound);
2499 // Redirect the inner exit to branch to outer condition.
2500 Instruction *Branch = &Cond->back();
2501 auto *BI = cast<BranchInst>(Branch);
2502 assert(BI->getSuccessor(1) == Exit)(static_cast <bool> (BI->getSuccessor(1) == Exit) ? void
(0) : __assert_fail ("BI->getSuccessor(1) == Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2502, __extension__ __PRETTY_FUNCTION__))
;
2503 BI->setSuccessor(1, OuterCond);
2504
2505 // Call the "fini" function if "ordered" is present in wsloop directive.
2506 if (Ordered) {
2507 Builder.SetInsertPoint(&Latch->back());
2508 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(IVTy, M, *this);
2509 Builder.CreateCall(DynamicFini, {SrcLoc, ThreadNum});
2510 }
2511
2512 // Add the barrier if requested.
2513 if (NeedsBarrier) {
2514 Builder.SetInsertPoint(&Exit->back());
2515 createBarrier(LocationDescription(Builder.saveIP(), DL),
2516 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
2517 /* CheckCancelFlag */ false);
2518 }
2519
2520 CLI->invalidate();
2521 return AfterIP;
2522}
2523
2524/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
2525/// after this \p OldTarget will be orphaned.
2526static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
2527 BasicBlock *NewTarget, DebugLoc DL) {
2528 for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
2529 redirectTo(Pred, NewTarget, DL);
2530}
2531
2532/// Determine which blocks in \p BBs are reachable from outside and remove the
2533/// ones that are not reachable from the function.
2534static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
2535 SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
2536 auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
2537 for (Use &U : BB->uses()) {
2538 auto *UseInst = dyn_cast<Instruction>(U.getUser());
2539 if (!UseInst)
2540 continue;
2541 if (BBsToErase.count(UseInst->getParent()))
2542 continue;
2543 return true;
2544 }
2545 return false;
2546 };
2547
2548 while (true) {
2549 bool Changed = false;
2550 for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
2551 if (HasRemainingUses(BB)) {
2552 BBsToErase.erase(BB);
2553 Changed = true;
2554 }
2555 }
2556 if (!Changed)
2557 break;
2558 }
2559
2560 SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
2561 DeleteDeadBlocks(BBVec);
2562}
2563
2564CanonicalLoopInfo *
2565OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
2566 InsertPointTy ComputeIP) {
2567 assert(Loops.size() >= 1 && "At least one loop required")(static_cast <bool> (Loops.size() >= 1 && "At least one loop required"
) ? void (0) : __assert_fail ("Loops.size() >= 1 && \"At least one loop required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2567, __extension__
__PRETTY_FUNCTION__))
;
2568 size_t NumLoops = Loops.size();
2569
2570 // Nothing to do if there is already just one loop.
2571 if (NumLoops == 1)
2572 return Loops.front();
2573
2574 CanonicalLoopInfo *Outermost = Loops.front();
2575 CanonicalLoopInfo *Innermost = Loops.back();
2576 BasicBlock *OrigPreheader = Outermost->getPreheader();
2577 BasicBlock *OrigAfter = Outermost->getAfter();
2578 Function *F = OrigPreheader->getParent();
2579
2580 // Loop control blocks that may become orphaned later.
2581 SmallVector<BasicBlock *, 12> OldControlBBs;
2582 OldControlBBs.reserve(6 * Loops.size());
2583 for (CanonicalLoopInfo *Loop : Loops)
2584 Loop->collectControlBlocks(OldControlBBs);
2585
2586 // Setup the IRBuilder for inserting the trip count computation.
2587 Builder.SetCurrentDebugLocation(DL);
2588 if (ComputeIP.isSet())
2589 Builder.restoreIP(ComputeIP);
2590 else
2591 Builder.restoreIP(Outermost->getPreheaderIP());
2592
2593 // Derive the collapsed' loop trip count.
2594 // TODO: Find common/largest indvar type.
2595 Value *CollapsedTripCount = nullptr;
2596 for (CanonicalLoopInfo *L : Loops) {
2597 assert(L->isValid() &&(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2598, __extension__
__PRETTY_FUNCTION__))
2598 "All loops to collapse must be valid canonical loops")(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2598, __extension__
__PRETTY_FUNCTION__))
;
2599 Value *OrigTripCount = L->getTripCount();
2600 if (!CollapsedTripCount) {
2601 CollapsedTripCount = OrigTripCount;
2602 continue;
2603 }
2604
2605 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
2606 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
2607 {}, /*HasNUW=*/true);
2608 }
2609
2610 // Create the collapsed loop control flow.
2611 CanonicalLoopInfo *Result =
2612 createLoopSkeleton(DL, CollapsedTripCount, F,
2613 OrigPreheader->getNextNode(), OrigAfter, "collapsed");
2614
2615 // Build the collapsed loop body code.
2616 // Start with deriving the input loop induction variables from the collapsed
2617 // one, using a divmod scheme. To preserve the original loops' order, the
2618 // innermost loop use the least significant bits.
2619 Builder.restoreIP(Result->getBodyIP());
2620
2621 Value *Leftover = Result->getIndVar();
2622 SmallVector<Value *> NewIndVars;
2623 NewIndVars.resize(NumLoops);
2624 for (int i = NumLoops - 1; i >= 1; --i) {
2625 Value *OrigTripCount = Loops[i]->getTripCount();
2626
2627 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
2628 NewIndVars[i] = NewIndVar;
2629
2630 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
2631 }
2632 // Outermost loop gets all the remaining bits.
2633 NewIndVars[0] = Leftover;
2634
2635 // Construct the loop body control flow.
2636 // We progressively construct the branch structure following in direction of
2637 // the control flow, from the leading in-between code, the loop nest body, the
2638 // trailing in-between code, and rejoining the collapsed loop's latch.
2639 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
2640 // the ContinueBlock is set, continue with that block. If ContinuePred, use
2641 // its predecessors as sources.
2642 BasicBlock *ContinueBlock = Result->getBody();
2643 BasicBlock *ContinuePred = nullptr;
2644 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
2645 BasicBlock *NextSrc) {
2646 if (ContinueBlock)
2647 redirectTo(ContinueBlock, Dest, DL);
2648 else
2649 redirectAllPredecessorsTo(ContinuePred, Dest, DL);
2650
2651 ContinueBlock = nullptr;
2652 ContinuePred = NextSrc;
2653 };
2654
2655 // The code before the nested loop of each level.
2656 // Because we are sinking it into the nest, it will be executed more often
2657 // that the original loop. More sophisticated schemes could keep track of what
2658 // the in-between code is and instantiate it only once per thread.
2659 for (size_t i = 0; i < NumLoops - 1; ++i)
2660 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
2661
2662 // Connect the loop nest body.
2663 ContinueWith(Innermost->getBody(), Innermost->getLatch());
2664
2665 // The code after the nested loop at each level.
2666 for (size_t i = NumLoops - 1; i > 0; --i)
2667 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
2668
2669 // Connect the finished loop to the collapsed loop latch.
2670 ContinueWith(Result->getLatch(), nullptr);
2671
2672 // Replace the input loops with the new collapsed loop.
2673 redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
2674 redirectTo(Result->getAfter(), Outermost->getAfter(), DL);
2675
2676 // Replace the input loop indvars with the derived ones.
2677 for (size_t i = 0; i < NumLoops; ++i)
2678 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
2679
2680 // Remove unused parts of the input loops.
2681 removeUnusedBlocksFromParent(OldControlBBs);
2682
2683 for (CanonicalLoopInfo *L : Loops)
2684 L->invalidate();
2685
2686#ifndef NDEBUG
2687 Result->assertOK();
2688#endif
2689 return Result;
2690}
2691
2692std::vector<CanonicalLoopInfo *>
2693OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
2694 ArrayRef<Value *> TileSizes) {
2695 assert(TileSizes.size() == Loops.size() &&(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2696, __extension__
__PRETTY_FUNCTION__))
2696 "Must pass as many tile sizes as there are loops")(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2696, __extension__
__PRETTY_FUNCTION__))
;
2697 int NumLoops = Loops.size();
2698 assert(NumLoops >= 1 && "At least one loop to tile required")(static_cast <bool> (NumLoops >= 1 && "At least one loop to tile required"
) ? void (0) : __assert_fail ("NumLoops >= 1 && \"At least one loop to tile required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2698, __extension__
__PRETTY_FUNCTION__))
;
2699
2700 CanonicalLoopInfo *OutermostLoop = Loops.front();
2701 CanonicalLoopInfo *InnermostLoop = Loops.back();
2702 Function *F = OutermostLoop->getBody()->getParent();
2703 BasicBlock *InnerEnter = InnermostLoop->getBody();
2704 BasicBlock *InnerLatch = InnermostLoop->getLatch();
2705
2706 // Loop control blocks that may become orphaned later.
2707 SmallVector<BasicBlock *, 12> OldControlBBs;
2708 OldControlBBs.reserve(6 * Loops.size());
2709 for (CanonicalLoopInfo *Loop : Loops)
2710 Loop->collectControlBlocks(OldControlBBs);
2711
2712 // Collect original trip counts and induction variable to be accessible by
2713 // index. Also, the structure of the original loops is not preserved during
2714 // the construction of the tiled loops, so do it before we scavenge the BBs of
2715 // any original CanonicalLoopInfo.
2716 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
2717 for (CanonicalLoopInfo *L : Loops) {
2718 assert(L->isValid() && "All input loops must be valid canonical loops")(static_cast <bool> (L->isValid() && "All input loops must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All input loops must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2718, __extension__
__PRETTY_FUNCTION__))
;
2719 OrigTripCounts.push_back(L->getTripCount());
2720 OrigIndVars.push_back(L->getIndVar());
2721 }
2722
2723 // Collect the code between loop headers. These may contain SSA definitions
2724 // that are used in the loop nest body. To be usable with in the innermost
2725 // body, these BasicBlocks will be sunk into the loop nest body. That is,
2726 // these instructions may be executed more often than before the tiling.
2727 // TODO: It would be sufficient to only sink them into body of the
2728 // corresponding tile loop.
2729 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
2730 for (int i = 0; i < NumLoops - 1; ++i) {
2731 CanonicalLoopInfo *Surrounding = Loops[i];
2732 CanonicalLoopInfo *Nested = Loops[i + 1];
2733
2734 BasicBlock *EnterBB = Surrounding->getBody();
2735 BasicBlock *ExitBB = Nested->getHeader();
2736 InbetweenCode.emplace_back(EnterBB, ExitBB);
2737 }
2738
2739 // Compute the trip counts of the floor loops.
2740 Builder.SetCurrentDebugLocation(DL);
2741 Builder.restoreIP(OutermostLoop->getPreheaderIP());
2742 SmallVector<Value *, 4> FloorCount, FloorRems;
2743 for (int i = 0; i < NumLoops; ++i) {
2744 Value *TileSize = TileSizes[i];
2745 Value *OrigTripCount = OrigTripCounts[i];
2746 Type *IVType = OrigTripCount->getType();
2747
2748 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
2749 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
2750
2751 // 0 if tripcount divides the tilesize, 1 otherwise.
2752 // 1 means we need an additional iteration for a partial tile.
2753 //
2754 // Unfortunately we cannot just use the roundup-formula
2755 // (tripcount + tilesize - 1)/tilesize
2756 // because the summation might overflow. We do not want introduce undefined
2757 // behavior when the untiled loop nest did not.
2758 Value *FloorTripOverflow =
2759 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
2760
2761 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
2762 FloorTripCount =
2763 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
2764 "omp_floor" + Twine(i) + ".tripcount", true);
2765
2766 // Remember some values for later use.
2767 FloorCount.push_back(FloorTripCount);
2768 FloorRems.push_back(FloorTripRem);
2769 }
2770
2771 // Generate the new loop nest, from the outermost to the innermost.
2772 std::vector<CanonicalLoopInfo *> Result;
2773 Result.reserve(NumLoops * 2);
2774
2775 // The basic block of the surrounding loop that enters the nest generated
2776 // loop.
2777 BasicBlock *Enter = OutermostLoop->getPreheader();
2778
2779 // The basic block of the surrounding loop where the inner code should
2780 // continue.
2781 BasicBlock *Continue = OutermostLoop->getAfter();
2782
2783 // Where the next loop basic block should be inserted.
2784 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
2785
2786 auto EmbeddNewLoop =
2787 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
2788 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
2789 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
2790 DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
2791 redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
2792 redirectTo(EmbeddedLoop->getAfter(), Continue, DL);
2793
2794 // Setup the position where the next embedded loop connects to this loop.
2795 Enter = EmbeddedLoop->getBody();
2796 Continue = EmbeddedLoop->getLatch();
2797 OutroInsertBefore = EmbeddedLoop->getLatch();
2798 return EmbeddedLoop;
2799 };
2800
2801 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
2802 const Twine &NameBase) {
2803 for (auto P : enumerate(TripCounts)) {
2804 CanonicalLoopInfo *EmbeddedLoop =
2805 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
2806 Result.push_back(EmbeddedLoop);
2807 }
2808 };
2809
2810 EmbeddNewLoops(FloorCount, "floor");
2811
2812 // Within the innermost floor loop, emit the code that computes the tile
2813 // sizes.
2814 Builder.SetInsertPoint(Enter->getTerminator());
2815 SmallVector<Value *, 4> TileCounts;
2816 for (int i = 0; i < NumLoops; ++i) {
2817 CanonicalLoopInfo *FloorLoop = Result[i];
2818 Value *TileSize = TileSizes[i];
2819
2820 Value *FloorIsEpilogue =
2821 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
2822 Value *TileTripCount =
2823 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
2824
2825 TileCounts.push_back(TileTripCount);
2826 }
2827
2828 // Create the tile loops.
2829 EmbeddNewLoops(TileCounts, "tile");
2830
2831 // Insert the inbetween code into the body.
2832 BasicBlock *BodyEnter = Enter;
2833 BasicBlock *BodyEntered = nullptr;
2834 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
2835 BasicBlock *EnterBB = P.first;
2836 BasicBlock *ExitBB = P.second;
2837
2838 if (BodyEnter)
2839 redirectTo(BodyEnter, EnterBB, DL);
2840 else
2841 redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);
2842
2843 BodyEnter = nullptr;
2844 BodyEntered = ExitBB;
2845 }
2846
2847 // Append the original loop nest body into the generated loop nest body.
2848 if (BodyEnter)
2849 redirectTo(BodyEnter, InnerEnter, DL);
2850 else
2851 redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
2852 redirectAllPredecessorsTo(InnerLatch, Continue, DL);
2853
2854 // Replace the original induction variable with an induction variable computed
2855 // from the tile and floor induction variables.
2856 Builder.restoreIP(Result.back()->getBodyIP());
2857 for (int i = 0; i < NumLoops; ++i) {
2858 CanonicalLoopInfo *FloorLoop = Result[i];
2859 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
2860 Value *OrigIndVar = OrigIndVars[i];
2861 Value *Size = TileSizes[i];
2862
2863 Value *Scale =
2864 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
2865 Value *Shift =
2866 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
2867 OrigIndVar->replaceAllUsesWith(Shift);
2868 }
2869
2870 // Remove unused parts of the original loops.
2871 removeUnusedBlocksFromParent(OldControlBBs);
2872
2873 for (CanonicalLoopInfo *L : Loops)
2874 L->invalidate();
2875
2876#ifndef NDEBUG
2877 for (CanonicalLoopInfo *GenL : Result)
2878 GenL->assertOK();
2879#endif
2880 return Result;
2881}
2882
2883/// Attach metadata \p Properties to the basic block described by \p BB. If the
2884/// basic block already has metadata, the basic block properties are appended.
2885static void addBasicBlockMetadata(BasicBlock *BB,
2886 ArrayRef<Metadata *> Properties) {
2887 // Nothing to do if no property to attach.
2888 if (Properties.empty())
2889 return;
2890
2891 LLVMContext &Ctx = BB->getContext();
2892 SmallVector<Metadata *> NewProperties;
2893 NewProperties.push_back(nullptr);
2894
2895 // If the basic block already has metadata, prepend it to the new metadata.
2896 MDNode *Existing = BB->getTerminator()->getMetadata(LLVMContext::MD_loop);
2897 if (Existing)
2898 append_range(NewProperties, drop_begin(Existing->operands(), 1));
2899
2900 append_range(NewProperties, Properties);
2901 MDNode *BasicBlockID = MDNode::getDistinct(Ctx, NewProperties);
2902 BasicBlockID->replaceOperandWith(0, BasicBlockID);
2903
2904 BB->getTerminator()->setMetadata(LLVMContext::MD_loop, BasicBlockID);
2905}
2906
2907/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
2908/// loop already has metadata, the loop properties are appended.
2909static void addLoopMetadata(CanonicalLoopInfo *Loop,
2910 ArrayRef<Metadata *> Properties) {
2911 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo")(static_cast <bool> (Loop->isValid() && "Expecting a valid CanonicalLoopInfo"
) ? void (0) : __assert_fail ("Loop->isValid() && \"Expecting a valid CanonicalLoopInfo\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2911, __extension__
__PRETTY_FUNCTION__))
;
2912
2913 // Attach metadata to the loop's latch
2914 BasicBlock *Latch = Loop->getLatch();
2915 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch")(static_cast <bool> (Latch && "A valid CanonicalLoopInfo must have a unique latch"
) ? void (0) : __assert_fail ("Latch && \"A valid CanonicalLoopInfo must have a unique latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2915, __extension__
__PRETTY_FUNCTION__))
;
2916 addBasicBlockMetadata(Latch, Properties);
2917}
2918
2919/// Attach llvm.access.group metadata to the memref instructions of \p Block
2920static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup,
2921 LoopInfo &LI) {
2922 for (Instruction &I : *Block) {
2923 if (I.mayReadOrWriteMemory()) {
2924 // TODO: This instruction may already have access group from
2925 // other pragmas e.g. #pragma clang loop vectorize. Append
2926 // so that the existing metadata is not overwritten.
2927 I.setMetadata(LLVMContext::MD_access_group, AccessGroup);
2928 }
2929 }
2930}
2931
2932void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
2933 LLVMContext &Ctx = Builder.getContext();
2934 addLoopMetadata(
2935 Loop, {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2936 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))});
2937}
2938
2939void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
2940 LLVMContext &Ctx = Builder.getContext();
2941 addLoopMetadata(
2942 Loop, {
2943 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2944 });
2945}
2946
2947void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
2948 Value *IfCond, ValueToValueMapTy &VMap,
2949 const Twine &NamePrefix) {
2950 Function *F = CanonicalLoop->getFunction();
2951
2952 // Define where if branch should be inserted
2953 Instruction *SplitBefore;
2954 if (Instruction::classof(IfCond)) {
2955 SplitBefore = dyn_cast<Instruction>(IfCond);
2956 } else {
2957 SplitBefore = CanonicalLoop->getPreheader()->getTerminator();
2958 }
2959
2960 // TODO: We should not rely on pass manager. Currently we use pass manager
2961 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
2962 // object. We should have a method which returns all blocks between
2963 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
2964 FunctionAnalysisManager FAM;
2965 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
2966 FAM.registerPass([]() { return LoopAnalysis(); });
2967 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
2968
2969 // Get the loop which needs to be cloned
2970 LoopAnalysis LIA;
2971 LoopInfo &&LI = LIA.run(*F, FAM);
2972 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());
2973
2974 // Create additional blocks for the if statement
2975 BasicBlock *Head = SplitBefore->getParent();
2976 Instruction *HeadOldTerm = Head->getTerminator();
2977 llvm::LLVMContext &C = Head->getContext();
2978 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
2979 C, NamePrefix + ".if.then", Head->getParent(), Head->getNextNode());
2980 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
2981 C, NamePrefix + ".if.else", Head->getParent(), CanonicalLoop->getExit());
2982
2983 // Create if condition branch.
2984 Builder.SetInsertPoint(HeadOldTerm);
2985 Instruction *BrInstr =
2986 Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);
2987 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
2988 // Then block contains branch to omp loop which needs to be vectorized
2989 spliceBB(IP, ThenBlock, false);
2990 ThenBlock->replaceSuccessorsPhiUsesWith(Head, ThenBlock);
2991
2992 Builder.SetInsertPoint(ElseBlock);
2993
2994 // Clone loop for the else branch
2995 SmallVector<BasicBlock *, 8> NewBlocks;
2996
2997 VMap[CanonicalLoop->getPreheader()] = ElseBlock;
2998 for (BasicBlock *Block : L->getBlocks()) {
2999 BasicBlock *NewBB = CloneBasicBlock(Block, VMap, "", F);
3000 NewBB->moveBefore(CanonicalLoop->getExit());
3001 VMap[Block] = NewBB;
3002 NewBlocks.push_back(NewBB);
3003 }
3004 remapInstructionsInBlocks(NewBlocks, VMap);
3005 Builder.CreateBr(NewBlocks.front());
3006}
3007
3008void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, Value *IfCond,
3009 ConstantInt *Simdlen, ConstantInt *Safelen) {
3010 LLVMContext &Ctx = Builder.getContext();
3011
3012 Function *F = CanonicalLoop->getFunction();
3013
3014 // TODO: We should not rely on pass manager. Currently we use pass manager
3015 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
3016 // object. We should have a method which returns all blocks between
3017 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
3018 FunctionAnalysisManager FAM;
3019 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
3020 FAM.registerPass([]() { return LoopAnalysis(); });
3021 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
3022
3023 LoopAnalysis LIA;
3024 LoopInfo &&LI = LIA.run(*F, FAM);
3025
3026 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());
3027
3028 if (IfCond) {
3029 ValueToValueMapTy VMap;
3030 createIfVersion(CanonicalLoop, IfCond, VMap, "simd");
3031 // Add metadata to the cloned loop which disables vectorization
3032 Value *MappedLatch = VMap.lookup(CanonicalLoop->getLatch());
3033 assert(MappedLatch &&(static_cast <bool> (MappedLatch && "Cannot find value which corresponds to original loop latch"
) ? void (0) : __assert_fail ("MappedLatch && \"Cannot find value which corresponds to original loop latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3034, __extension__
__PRETTY_FUNCTION__))
3034 "Cannot find value which corresponds to original loop latch")(static_cast <bool> (MappedLatch && "Cannot find value which corresponds to original loop latch"
) ? void (0) : __assert_fail ("MappedLatch && \"Cannot find value which corresponds to original loop latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3034, __extension__
__PRETTY_FUNCTION__))
;
3035 assert(isa<BasicBlock>(MappedLatch) &&(static_cast <bool> (isa<BasicBlock>(MappedLatch)
&& "Cannot cast mapped latch block value to BasicBlock"
) ? void (0) : __assert_fail ("isa<BasicBlock>(MappedLatch) && \"Cannot cast mapped latch block value to BasicBlock\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
3036 "Cannot cast mapped latch block value to BasicBlock")(static_cast <bool> (isa<BasicBlock>(MappedLatch)
&& "Cannot cast mapped latch block value to BasicBlock"
) ? void (0) : __assert_fail ("isa<BasicBlock>(MappedLatch) && \"Cannot cast mapped latch block value to BasicBlock\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
;
3037 BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(MappedLatch);
3038 ConstantAsMetadata *BoolConst =
3039 ConstantAsMetadata::get(ConstantInt::getFalse(Type::getInt1Ty(Ctx)));
3040 addBasicBlockMetadata(
3041 NewLatchBlock,
3042 {MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
3043 BoolConst})});
3044 }
3045
3046 SmallSet<BasicBlock *, 8> Reachable;
3047
3048 // Get the basic blocks from the loop in which memref instructions
3049 // can be found.
3050 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
3051 // preferably without running any passes.
3052 for (BasicBlock *Block : L->getBlocks()) {
3053 if (Block == CanonicalLoop->getCond() ||
3054 Block == CanonicalLoop->getHeader())
3055 continue;
3056 Reachable.insert(Block);
3057 }
3058
3059 SmallVector<Metadata *> LoopMDList;
3060
3061 // In presence of finite 'safelen', it may be unsafe to mark all
3062 // the memory instructions parallel, because loop-carried
3063 // dependences of 'safelen' iterations are possible.
3064 if (Safelen == nullptr) {
3065 // Add access group metadata to memory-access instructions.
3066 MDNode *AccessGroup = MDNode::getDistinct(Ctx, {});
3067 for (BasicBlock *BB : Reachable)
3068 addSimdMetadata(BB, AccessGroup, LI);
3069 // TODO: If the loop has existing parallel access metadata, have
3070 // to combine two lists.
3071 LoopMDList.push_back(MDNode::get(
3072 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccessGroup}));
3073 }
3074
3075 // Use the above access group metadata to create loop level
3076 // metadata, which should be distinct for each loop.
3077 ConstantAsMetadata *BoolConst =
3078 ConstantAsMetadata::get(ConstantInt::getTrue(Type::getInt1Ty(Ctx)));
3079 LoopMDList.push_back(MDNode::get(
3080 Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), BoolConst}));
3081
3082 if (Simdlen || Safelen) {
3083 // If both simdlen and safelen clauses are specified, the value of the
3084 // simdlen parameter must be less than or equal to the value of the safelen
3085 // parameter. Therefore, use safelen only in the absence of simdlen.
3086 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
3087 LoopMDList.push_back(
3088 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.width"),
3089 ConstantAsMetadata::get(VectorizeWidth)}));
3090 }
3091
3092 addLoopMetadata(CanonicalLoop, LoopMDList);
3093}
3094
3095/// Create the TargetMachine object to query the backend for optimization
3096/// preferences.
3097///
3098/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
3099/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
3100/// needed for the LLVM pass pipline. We use some default options to avoid
3101/// having to pass too many settings from the frontend that probably do not
3102/// matter.
3103///
3104/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
3105/// method. If we are going to use TargetMachine for more purposes, especially
3106/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
3107/// might become be worth requiring front-ends to pass on their TargetMachine,
3108/// or at least cache it between methods. Note that while fontends such as Clang
3109/// have just a single main TargetMachine per translation unit, "target-cpu" and
3110/// "target-features" that determine the TargetMachine are per-function and can
3111/// be overrided using __attribute__((target("OPTIONS"))).
3112static std::unique_ptr<TargetMachine>
3113createTargetMachine(Function *F, CodeGenOpt::Level OptLevel) {
3114 Module *M = F->getParent();
3115
3116 StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString();
3117 StringRef Features = F->getFnAttribute("target-features").getValueAsString();
3118 const std::string &Triple = M->getTargetTriple();
3119
3120 std::string Error;
3121 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
3122 if (!TheTarget)
3123 return {};
3124
3125 llvm::TargetOptions Options;
3126 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
3127 Triple, CPU, Features, Options, /*RelocModel=*/None, /*CodeModel=*/None,
3128 OptLevel));
3129}
3130
3131/// Heuristically determine the best-performant unroll factor for \p CLI. This
3132/// depends on the target processor. We are re-using the same heuristics as the
3133/// LoopUnrollPass.
3134static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
3135 Function *F = CLI->getFunction();
3136
3137 // Assume the user requests the most aggressive unrolling, even if the rest of
3138 // the code is optimized using a lower setting.
3139 CodeGenOpt::Level OptLevel = CodeGenOpt::Aggressive;
3140 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
3141
3142 FunctionAnalysisManager FAM;
3143 FAM.registerPass([]() { return TargetLibraryAnalysis(); });
3144 FAM.registerPass([]() { return AssumptionAnalysis(); });
3145 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
3146 FAM.registerPass([]() { return LoopAnalysis(); });
3147 FAM.registerPass([]() { return ScalarEvolutionAnalysis(); });
3148 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
3149 TargetIRAnalysis TIRA;
3150 if (TM)
3151 TIRA = TargetIRAnalysis(
3152 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
3153 FAM.registerPass([&]() { return TIRA; });
3154
3155 TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM);
3156 ScalarEvolutionAnalysis SEA;
3157 ScalarEvolution &&SE = SEA.run(*F, FAM);
3158 DominatorTreeAnalysis DTA;
3159 DominatorTree &&DT = DTA.run(*F, FAM);
3160 LoopAnalysis LIA;
3161 LoopInfo &&LI = LIA.run(*F, FAM);
3162 AssumptionAnalysis ACT;
3163 AssumptionCache &&AC = ACT.run(*F, FAM);
3164 OptimizationRemarkEmitter ORE{F};
3165
3166 Loop *L = LI.getLoopFor(CLI->getHeader());
3167 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop")(static_cast <bool> (L && "Expecting CanonicalLoopInfo to be recognized as a loop"
) ? void (0) : __assert_fail ("L && \"Expecting CanonicalLoopInfo to be recognized as a loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3167, __extension__
__PRETTY_FUNCTION__))
;
3168
3169 TargetTransformInfo::UnrollingPreferences UP =
3170 gatherUnrollingPreferences(L, SE, TTI,
3171 /*BlockFrequencyInfo=*/nullptr,
3172 /*ProfileSummaryInfo=*/nullptr, ORE, OptLevel,
3173 /*UserThreshold=*/None,
3174 /*UserCount=*/None,
3175 /*UserAllowPartial=*/true,
3176 /*UserAllowRuntime=*/true,
3177 /*UserUpperBound=*/None,
3178 /*UserFullUnrollMaxCount=*/None);
3179
3180 UP.Force = true;
3181
3182 // Account for additional optimizations taking place before the LoopUnrollPass
3183 // would unroll the loop.
3184 UP.Threshold *= UnrollThresholdFactor;
3185 UP.PartialThreshold *= UnrollThresholdFactor;
3186
3187 // Use normal unroll factors even if the rest of the code is optimized for
3188 // size.
3189 UP.OptSizeThreshold = UP.Threshold;
3190 UP.PartialOptSizeThreshold = UP.PartialThreshold;
3191
3192 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3193 << " Threshold=" << UP.Threshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3194 << " PartialThreshold=" << UP.PartialThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3195 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3196 << " PartialOptSizeThreshold="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3197 << UP.PartialOptSizeThreshold << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
;
3198
3199 // Disable peeling.
3200 TargetTransformInfo::PeelingPreferences PP =
3201 gatherPeelingPreferences(L, SE, TTI,
3202 /*UserAllowPeeling=*/false,
3203 /*UserAllowProfileBasedPeeling=*/false,
3204 /*UnrollingSpecficValues=*/false);
3205
3206 SmallPtrSet<const Value *, 32> EphValues;
3207 CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
3208
3209 // Assume that reads and writes to stack variables can be eliminated by
3210 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
3211 // size.
3212 for (BasicBlock *BB : L->blocks()) {
3213 for (Instruction &I : *BB) {
3214 Value *Ptr;
3215 if (auto *Load = dyn_cast<LoadInst>(&I)) {
3216 Ptr = Load->getPointerOperand();
3217 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
3218 Ptr = Store->getPointerOperand();
3219 } else
3220 continue;
3221
3222 Ptr = Ptr->stripPointerCasts();
3223
3224 if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) {
3225 if (Alloca->getParent() == &F->getEntryBlock())
3226 EphValues.insert(&I);
3227 }
3228 }
3229 }
3230
3231 unsigned NumInlineCandidates;
3232 bool NotDuplicatable;
3233 bool Convergent;
3234 InstructionCost LoopSizeIC =
3235 ApproximateLoopSize(L, NumInlineCandidates, NotDuplicatable, Convergent,
3236 TTI, EphValues, UP.BEInsns);
3237 LLVM_DEBUG(dbgs() << "Estimated loop size is " << LoopSizeIC << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Estimated loop size is "
<< LoopSizeIC << "\n"; } } while (false)
;
3238
3239 // Loop is not unrollable if the loop contains certain instructions.
3240 if (NotDuplicatable || Convergent || !LoopSizeIC.isValid()) {
3241 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Loop not considered unrollable\n"
; } } while (false)
;
3242 return 1;
3243 }
3244 unsigned LoopSize = *LoopSizeIC.getValue();
3245
3246 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
3247 // be able to use it.
3248 int TripCount = 0;
3249 int MaxTripCount = 0;
3250 bool MaxOrZero = false;
3251 unsigned TripMultiple = 0;
3252
3253 bool UseUpperBound = false;
3254 computeUnrollCount(L, TTI, DT, &LI, &AC, SE, EphValues, &ORE, TripCount,
3255 MaxTripCount, MaxOrZero, TripMultiple, LoopSize, UP, PP,
3256 UseUpperBound);
3257 unsigned Factor = UP.Count;
3258 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Suggesting unroll factor of "
<< Factor << "\n"; } } while (false)
;
3259
3260 // This function returns 1 to signal to not unroll a loop.
3261 if (Factor == 0)
3262 return 1;
3263 return Factor;
3264}
3265
3266void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
3267 int32_t Factor,
3268 CanonicalLoopInfo **UnrolledCLI) {
3269 assert(Factor >= 0 && "Unroll factor must not be negative")(static_cast <bool> (Factor >= 0 && "Unroll factor must not be negative"
) ? void (0) : __assert_fail ("Factor >= 0 && \"Unroll factor must not be negative\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3269, __extension__
__PRETTY_FUNCTION__))
;
3270
3271 Function *F = Loop->getFunction();
3272 LLVMContext &Ctx = F->getContext();
3273
3274 // If the unrolled loop is not used for another loop-associated directive, it
3275 // is sufficient to add metadata for the LoopUnrollPass.
3276 if (!UnrolledCLI) {
3277 SmallVector<Metadata *, 2> LoopMetadata;
3278 LoopMetadata.push_back(
3279 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")));
3280
3281 if (Factor >= 1) {
3282 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3283 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
3284 LoopMetadata.push_back(MDNode::get(
3285 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst}));
3286 }
3287
3288 addLoopMetadata(Loop, LoopMetadata);
3289 return;
3290 }
3291
3292 // Heuristically determine the unroll factor.
3293 if (Factor == 0)
3294 Factor = computeHeuristicUnrollFactor(Loop);
3295
3296 // No change required with unroll factor 1.
3297 if (Factor == 1) {
3298 *UnrolledCLI = Loop;
3299 return;
3300 }
3301
3302 assert(Factor >= 2 &&(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3303, __extension__
__PRETTY_FUNCTION__))
3303 "unrolling only makes sense with a factor of 2 or larger")(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3303, __extension__
__PRETTY_FUNCTION__))
;
3304
3305 Type *IndVarTy = Loop->getIndVarType();
3306
3307 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
3308 // unroll the inner loop.
3309 Value *FactorVal =
3310 ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor,
3311 /*isSigned=*/false));
3312 std::vector<CanonicalLoopInfo *> LoopNest =
3313 tileLoops(DL, {Loop}, {FactorVal});
3314 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling")(static_cast <bool> (LoopNest.size() == 2 && "Expect 2 loops after tiling"
) ? void (0) : __assert_fail ("LoopNest.size() == 2 && \"Expect 2 loops after tiling\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3314, __extension__
__PRETTY_FUNCTION__))
;
3315 *UnrolledCLI = LoopNest[0];
3316 CanonicalLoopInfo *InnerLoop = LoopNest[1];
3317
3318 // LoopUnrollPass can only fully unroll loops with constant trip count.
3319 // Unroll by the unroll factor with a fallback epilog for the remainder
3320 // iterations if necessary.
3321 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3322 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
3323 addLoopMetadata(
3324 InnerLoop,
3325 {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
3326 MDNode::get(
3327 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst})});
3328
3329#ifndef NDEBUG
3330 (*UnrolledCLI)->assertOK();
3331#endif
3332}
3333
3334OpenMPIRBuilder::InsertPointTy
3335OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
3336 llvm::Value *BufSize, llvm::Value *CpyBuf,
3337 llvm::Value *CpyFn, llvm::Value *DidIt) {
3338 if (!updateToLocation(Loc))
3339 return Loc.IP;
3340
3341 uint32_t SrcLocStrSize;
3342 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3343 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3344 Value *ThreadId = getOrCreateThreadID(Ident);
3345
3346 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
3347
3348 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
3349
3350 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
3351 Builder.CreateCall(Fn, Args);
3352
3353 return Builder.saveIP();
3354}
3355
3356OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
3357 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3358 FinalizeCallbackTy FiniCB, bool IsNowait, llvm::Value *DidIt) {
3359
3360 if (!updateToLocation(Loc))
3361 return Loc.IP;
3362
3363 // If needed (i.e. not null), initialize `DidIt` with 0
3364 if (DidIt) {
3365 Builder.CreateStore(Builder.getInt32(0), DidIt);
3366 }
3367
3368 Directive OMPD = Directive::OMPD_single;
3369 uint32_t SrcLocStrSize;
3370 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3371 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3372 Value *ThreadId = getOrCreateThreadID(Ident);
3373 Value *Args[] = {Ident, ThreadId};
3374
3375 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
3376 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3377
3378 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
3379 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3380
3381 // generates the following:
3382 // if (__kmpc_single()) {
3383 // .... single region ...
3384 // __kmpc_end_single
3385 // }
3386 // __kmpc_barrier
3387
3388 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3389 /*Conditional*/ true,
3390 /*hasFinalize*/ true);
3391 if (!IsNowait)
3392 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
3393 omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
3394 /* CheckCancelFlag */ false);
3395 return Builder.saveIP();
3396}
3397
3398OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
3399 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3400 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
3401
3402 if (!updateToLocation(Loc))
3403 return Loc.IP;
3404
3405 Directive OMPD = Directive::OMPD_critical;
3406 uint32_t SrcLocStrSize;
3407 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3408 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3409 Value *ThreadId = getOrCreateThreadID(Ident);
3410 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
3411 Value *Args[] = {Ident, ThreadId, LockVar};
3412
3413 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
3414 Function *RTFn = nullptr;
3415 if (HintInst) {
3416 // Add Hint to entry Args and create call
3417 EnterArgs.push_back(HintInst);
3418 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
3419 } else {
3420 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
3421 }
3422 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
3423
3424 Function *ExitRTLFn =
3425 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
3426 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3427
3428 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3429 /*Conditional*/ false, /*hasFinalize*/ true);
3430}
3431
3432OpenMPIRBuilder::InsertPointTy
3433OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
3434 InsertPointTy AllocaIP, unsigned NumLoops,
3435 ArrayRef<llvm::Value *> StoreValues,
3436 const Twine &Name, bool IsDependSource) {
3437 assert((static_cast <bool> (llvm::all_of(StoreValues, [](Value
*SV) { return SV->getType()->isIntegerTy(64); }) &&
"OpenMP runtime requires depend vec with i64 type") ? void (
0) : __assert_fail ("llvm::all_of(StoreValues, [](Value *SV) { return SV->getType()->isIntegerTy(64); }) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
3438 llvm::all_of(StoreValues,(static_cast <bool> (llvm::all_of(StoreValues, [](Value
*SV) { return SV->getType()->isIntegerTy(64); }) &&
"OpenMP runtime requires depend vec with i64 type") ? void (
0) : __assert_fail ("llvm::all_of(StoreValues, [](Value *SV) { return SV->getType()->isIntegerTy(64); }) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
3439 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&(static_cast <bool> (llvm::all_of(StoreValues, [](Value
*SV) { return SV->getType()->isIntegerTy(64); }) &&
"OpenMP runtime requires depend vec with i64 type") ? void (
0) : __assert_fail ("llvm::all_of(StoreValues, [](Value *SV) { return SV->getType()->isIntegerTy(64); }) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
3440 "OpenMP runtime requires depend vec with i64 type")(static_cast <bool> (llvm::all_of(StoreValues, [](Value
*SV) { return SV->getType()->isIntegerTy(64); }) &&
"OpenMP runtime requires depend vec with i64 type") ? void (
0) : __assert_fail ("llvm::all_of(StoreValues, [](Value *SV) { return SV->getType()->isIntegerTy(64); }) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3440, __extension__
__PRETTY_FUNCTION__))
;
3441
3442 if (!updateToLocation(Loc))
3443 return Loc.IP;
3444
3445 // Allocate space for vector and generate alloc instruction.
3446 auto *ArrI64Ty = ArrayType::get(Int64, NumLoops);
3447 Builder.restoreIP(AllocaIP);
3448 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);
3449 ArgsBase->setAlignment(Align(8));
3450 Builder.restoreIP(Loc.IP);
3451
3452 // Store the index value with offset in depend vector.
3453 for (unsigned I = 0; I < NumLoops; ++I) {
3454 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
3455 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});
3456 StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);
3457 STInst->setAlignment(Align(8));
3458 }
3459
3460 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
3461 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});
3462
3463 uint32_t SrcLocStrSize;
3464 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3465 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3466 Value *ThreadId = getOrCreateThreadID(Ident);
3467 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
3468
3469 Function *RTLFn = nullptr;
3470 if (IsDependSource)
3471 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_post);
3472 else
3473 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_wait);
3474 Builder.CreateCall(RTLFn, Args);
3475
3476 return Builder.saveIP();
3477}
3478
3479OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
3480 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3481 FinalizeCallbackTy FiniCB, bool IsThreads) {
3482 if (!updateToLocation(Loc))
3483 return Loc.IP;
3484
3485 Directive OMPD = Directive::OMPD_ordered;
3486 Instruction *EntryCall = nullptr;
3487 Instruction *ExitCall = nullptr;
3488
3489 if (IsThreads) {
3490 uint32_t SrcLocStrSize;
3491 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3492 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3493 Value *ThreadId = getOrCreateThreadID(Ident);
3494 Value *Args[] = {Ident, ThreadId};
3495
3496 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_ordered);
3497 EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3498
3499 Function *ExitRTLFn =
3500 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_ordered);
3501 ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3502 }
3503
3504 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3505 /*Conditional*/ false, /*hasFinalize*/ true);
3506}
3507
3508OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
3509 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
3510 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
3511 bool HasFinalize, bool IsCancellable) {
3512
3513 if (HasFinalize)
3514 FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});
3515
3516 // Create inlined region's entry and body blocks, in preparation
3517 // for conditional creation
3518 BasicBlock *EntryBB = Builder.GetInsertBlock();
3519 Instruction *SplitPos = EntryBB->getTerminator();
3520 if (!isa_and_nonnull<BranchInst>(SplitPos))
3521 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
3522 BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
3523 BasicBlock *FiniBB =
3524 EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");
3525
3526 Builder.SetInsertPoint(EntryBB->getTerminator());
3527 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
3528
3529 // generate body
3530 BodyGenCB(/* AllocaIP */ InsertPointTy(),
3531 /* CodeGenIP */ Builder.saveIP());
3532
3533 // emit exit call and do any needed finalization.
3534 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
3535 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3537, __extension__
__PRETTY_FUNCTION__))
3536 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3537, __extension__
__PRETTY_FUNCTION__))
3537 "Unexpected control flow graph state!!")(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3537, __extension__
__PRETTY_FUNCTION__))
;
3538 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
3539 assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3540, __extension__
__PRETTY_FUNCTION__))
3540 "Unexpected Control Flow State!")(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3540, __extension__
__PRETTY_FUNCTION__))
;
3541 MergeBlockIntoPredecessor(FiniBB);
3542
3543 // If we are skipping the region of a non conditional, remove the exit
3544 // block, and clear the builder's insertion point.
3545 assert(SplitPos->getParent() == ExitBB &&(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3546, __extension__
__PRETTY_FUNCTION__))
3546 "Unexpected Insertion point location!")(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3546, __extension__
__PRETTY_FUNCTION__))
;
3547 auto merged = MergeBlockIntoPredecessor(ExitBB);
3548 BasicBlock *ExitPredBB = SplitPos->getParent();
3549 auto InsertBB = merged ? ExitPredBB : ExitBB;
3550 if (!isa_and_nonnull<BranchInst>(SplitPos))
3551 SplitPos->eraseFromParent();
3552 Builder.SetInsertPoint(InsertBB);
3553
3554 return Builder.saveIP();
3555}
3556
3557OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
3558 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
3559 // if nothing to do, Return current insertion point.
3560 if (!Conditional || !EntryCall)
3561 return Builder.saveIP();
3562
3563 BasicBlock *EntryBB = Builder.GetInsertBlock();
3564 Value *CallBool = Builder.CreateIsNotNull(EntryCall);
3565 auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
3566 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
3567
3568 // Emit thenBB and set the Builder's insertion point there for
3569 // body generation next. Place the block after the current block.
3570 Function *CurFn = EntryBB->getParent();
3571 CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);
3572
3573 // Move Entry branch to end of ThenBB, and replace with conditional
3574 // branch (If-stmt)
3575 Instruction *EntryBBTI = EntryBB->getTerminator();
3576 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
3577 EntryBBTI->removeFromParent();
3578 Builder.SetInsertPoint(UI);
3579 Builder.Insert(EntryBBTI);
3580 UI->eraseFromParent();
3581 Builder.SetInsertPoint(ThenBB->getTerminator());
3582
3583 // return an insertion point to ExitBB.
3584 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
3585}
3586
3587OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
3588 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
3589 bool HasFinalize) {
3590
3591 Builder.restoreIP(FinIP);
3592
3593 // If there is finalization to do, emit it before the exit call
3594 if (HasFinalize) {
3595 assert(!FinalizationStack.empty() &&(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3596, __extension__
__PRETTY_FUNCTION__))
3596 "Unexpected finalization stack state!")(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3596, __extension__
__PRETTY_FUNCTION__))
;
3597
3598 FinalizationInfo Fi = FinalizationStack.pop_back_val();
3599 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!")(static_cast <bool> (Fi.DK == OMPD && "Unexpected Directive for Finalization call!"
) ? void (0) : __assert_fail ("Fi.DK == OMPD && \"Unexpected Directive for Finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3599, __extension__
__PRETTY_FUNCTION__))
;
3600
3601 Fi.FiniCB(FinIP);
3602
3603 BasicBlock *FiniBB = FinIP.getBlock();
3604 Instruction *FiniBBTI = FiniBB->getTerminator();
3605
3606 // set Builder IP for call creation
3607 Builder.SetInsertPoint(FiniBBTI);
3608 }
3609
3610 if (!ExitCall)
3611 return Builder.saveIP();
3612
3613 // place the Exitcall as last instruction before Finalization block terminator
3614 ExitCall->removeFromParent();
3615 Builder.Insert(ExitCall);
3616
3617 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
3618 ExitCall->getIterator());
3619}
3620
3621OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
3622 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
3623 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
3624 if (!IP.isSet())
3625 return IP;
3626
3627 IRBuilder<>::InsertPointGuard IPG(Builder);
3628
3629 // creates the following CFG structure
3630 // OMP_Entry : (MasterAddr != PrivateAddr)?
3631 // F T
3632 // | \
3633 // | copin.not.master
3634 // | /
3635 // v /
3636 // copyin.not.master.end
3637 // |
3638 // v
3639 // OMP.Entry.Next
3640
3641 BasicBlock *OMP_Entry = IP.getBlock();
3642 Function *CurFn = OMP_Entry->getParent();
3643 BasicBlock *CopyBegin =
3644 BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
3645 BasicBlock *CopyEnd = nullptr;
3646
3647 // If entry block is terminated, split to preserve the branch to following
3648 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
3649 if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
3650 CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
3651 "copyin.not.master.end");
3652 OMP_Entry->getTerminator()->eraseFromParent();
3653 } else {
3654 CopyEnd =
3655 BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
3656 }
3657
3658 Builder.SetInsertPoint(OMP_Entry);
3659 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
3660 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
3661 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
3662 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
3663
3664 Builder.SetInsertPoint(CopyBegin);
3665 if (BranchtoEnd)
3666 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
3667
3668 return Builder.saveIP();
3669}
3670
3671CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
3672 Value *Size, Value *Allocator,
3673 std::string Name) {
3674 IRBuilder<>::InsertPointGuard IPG(Builder);
3675 Builder.restoreIP(Loc.IP);
3676
3677 uint32_t SrcLocStrSize;
3678 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3679 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3680 Value *ThreadId = getOrCreateThreadID(Ident);
3681 Value *Args[] = {ThreadId, Size, Allocator};
3682
3683 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
3684
3685 return Builder.CreateCall(Fn, Args, Name);
3686}
3687
3688CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
3689 Value *Addr, Value *Allocator,
3690 std::string Name) {
3691 IRBuilder<>::InsertPointGuard IPG(Builder);
3692 Builder.restoreIP(Loc.IP);
3693
3694 uint32_t SrcLocStrSize;
3695 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3696 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3697 Value *ThreadId = getOrCreateThreadID(Ident);
3698 Value *Args[] = {ThreadId, Addr, Allocator};
3699 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
3700 return Builder.CreateCall(Fn, Args, Name);
3701}
3702
3703CallInst *OpenMPIRBuilder::createOMPInteropInit(
3704 const LocationDescription &Loc, Value *InteropVar,
3705 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
3706 Value *DependenceAddress, bool HaveNowaitClause) {
3707 IRBuilder<>::InsertPointGuard IPG(Builder);
3708 Builder.restoreIP(Loc.IP);
3709
3710 uint32_t SrcLocStrSize;
3711 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3712 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3713 Value *ThreadId = getOrCreateThreadID(Ident);
3714 if (Device == nullptr)
3715 Device = ConstantInt::get(Int32, -1);
3716 Constant *InteropTypeVal = ConstantInt::get(Int64, (int)InteropType);
3717 if (NumDependences == nullptr) {
3718 NumDependences = ConstantInt::get(Int32, 0);
3719 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3720 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3721 }
3722 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3723 Value *Args[] = {
3724 Ident, ThreadId, InteropVar, InteropTypeVal,
3725 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
3726
3727 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_init);
3728
3729 return Builder.CreateCall(Fn, Args);
3730}
3731
3732CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
3733 const LocationDescription &Loc, Value *InteropVar, Value *Device,
3734 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
3735 IRBuilder<>::InsertPointGuard IPG(Builder);
3736 Builder.restoreIP(Loc.IP);
3737
3738 uint32_t SrcLocStrSize;
3739 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3740 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3741 Value *ThreadId = getOrCreateThreadID(Ident);
3742 if (Device == nullptr)
3743 Device = ConstantInt::get(Int32, -1);
3744 if (NumDependences == nullptr) {
3745 NumDependences = ConstantInt::get(Int32, 0);
3746 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3747 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3748 }
3749 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3750 Value *Args[] = {
3751 Ident, ThreadId, InteropVar, Device,
3752 NumDependences, DependenceAddress, HaveNowaitClauseVal};
3753
3754 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_destroy);
3755
3756 return Builder.CreateCall(Fn, Args);
3757}
3758
3759CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
3760 Value *InteropVar, Value *Device,
3761 Value *NumDependences,
3762 Value *DependenceAddress,
3763 bool HaveNowaitClause) {
3764 IRBuilder<>::InsertPointGuard IPG(Builder);
3765 Builder.restoreIP(Loc.IP);
3766 uint32_t SrcLocStrSize;
3767 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3768 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3769 Value *ThreadId = getOrCreateThreadID(Ident);
3770 if (Device == nullptr)
3771 Device = ConstantInt::get(Int32, -1);
3772 if (NumDependences == nullptr) {
3773 NumDependences = ConstantInt::get(Int32, 0);
3774 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3775 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3776 }
3777 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3778 Value *Args[] = {
3779 Ident, ThreadId, InteropVar, Device,
3780 NumDependences, DependenceAddress, HaveNowaitClauseVal};
3781
3782 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_use);
3783
3784 return Builder.CreateCall(Fn, Args);
3785}
3786
3787CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
3788 const LocationDescription &Loc, llvm::Value *Pointer,
3789 llvm::ConstantInt *Size, const llvm::Twine &Name) {
3790 IRBuilder<>::InsertPointGuard IPG(Builder);
3791 Builder.restoreIP(Loc.IP);
3792
3793 uint32_t SrcLocStrSize;
3794 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3795 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3796 Value *ThreadId = getOrCreateThreadID(Ident);
3797 Constant *ThreadPrivateCache =
3798 getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
3799 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
3800
3801 Function *Fn =
3802 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
3803
3804 return Builder.CreateCall(Fn, Args);
3805}
3806
3807OpenMPIRBuilder::InsertPointTy
3808OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
3809 bool RequiresFullRuntime) {
3810 if (!updateToLocation(Loc))
3811 return Loc.IP;
3812
3813 uint32_t SrcLocStrSize;
3814 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3815 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3816 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
3817 IntegerType::getInt8Ty(Int8->getContext()),
3818 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
3819 ConstantInt *UseGenericStateMachine =
3820 ConstantInt::getBool(Int32->getContext(), !IsSPMD);
3821 ConstantInt *RequiresFullRuntimeVal =
3822 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
3823
3824 Function *Fn = getOrCreateRuntimeFunctionPtr(
3825 omp::RuntimeFunction::OMPRTL___kmpc_target_init);
3826
3827 CallInst *ThreadKind = Builder.CreateCall(
3828 Fn, {Ident, IsSPMDVal, UseGenericStateMachine, RequiresFullRuntimeVal});
3829
3830 Value *ExecUserCode = Builder.CreateICmpEQ(
3831 ThreadKind, ConstantInt::get(ThreadKind->getType(), -1),
3832 "exec_user_code");
3833
3834 // ThreadKind = __kmpc_target_init(...)
3835 // if (ThreadKind == -1)
3836 // user_code
3837 // else
3838 // return;
3839
3840 auto *UI = Builder.CreateUnreachable();
3841 BasicBlock *CheckBB = UI->getParent();
3842 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry");
3843
3844 BasicBlock *WorkerExitBB = BasicBlock::Create(
3845 CheckBB->getContext(), "worker.exit", CheckBB->getParent());
3846 Builder.SetInsertPoint(WorkerExitBB);
3847 Builder.CreateRetVoid();
3848
3849 auto *CheckBBTI = CheckBB->getTerminator();
3850 Builder.SetInsertPoint(CheckBBTI);
3851 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);
3852
3853 CheckBBTI->eraseFromParent();
3854 UI->eraseFromParent();
3855
3856 // Continue in the "user_code" block, see diagram above and in
3857 // openmp/libomptarget/deviceRTLs/common/include/target.h .
3858 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
3859}
3860
3861void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
3862 bool IsSPMD,
3863 bool RequiresFullRuntime) {
3864 if (!updateToLocation(Loc))
3865 return;
3866
3867 uint32_t SrcLocStrSize;
3868 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3869 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3870 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
3871 IntegerType::getInt8Ty(Int8->getContext()),
3872 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
3873 ConstantInt *RequiresFullRuntimeVal =
3874 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
3875
3876 Function *Fn = getOrCreateRuntimeFunctionPtr(
3877 omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
3878
3879 Builder.CreateCall(Fn, {Ident, IsSPMDVal, RequiresFullRuntimeVal});
3880}
3881
3882std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
3883 StringRef FirstSeparator,
3884 StringRef Separator) {
3885 SmallString<128> Buffer;
3886 llvm::raw_svector_ostream OS(Buffer);
3887 StringRef Sep = FirstSeparator;
3888 for (StringRef Part : Parts) {
3889 OS << Sep << Part;
3890 Sep = Separator;
3891 }
3892 return OS.str().str();
3893}
3894
3895Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
3896 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3897 // TODO: Replace the twine arg with stringref to get rid of the conversion
3898 // logic. However This is taken from current implementation in clang as is.
3899 // Since this method is used in many places exclusively for OMP internal use
3900 // we will keep it as is for temporarily until we move all users to the
3901 // builder and then, if possible, fix it everywhere in one go.
3902 SmallString<256> Buffer;
3903 llvm::raw_svector_ostream Out(Buffer);
3904 Out << Name;
3905 StringRef RuntimeName = Out.str();
3906 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3907 if (Elem.second) {
3908 assert(cast<PointerType>(Elem.second->getType())(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3910, __extension__
__PRETTY_FUNCTION__))
3909 ->isOpaqueOrPointeeTypeMatches(Ty) &&(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3910, __extension__
__PRETTY_FUNCTION__))
3910 "OMP internal variable has different type than requested")(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3910, __extension__
__PRETTY_FUNCTION__))
;
3911 } else {
3912 // TODO: investigate the appropriate linkage type used for the global
3913 // variable for possibly changing that to internal or private, or maybe
3914 // create different versions of the function for different OMP internal
3915 // variables.
3916 Elem.second = new llvm::GlobalVariable(
3917 M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
3918 llvm::Constant::getNullValue(Ty), Elem.first(),
3919 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
3920 AddressSpace);
3921 }
3922
3923 return Elem.second;
3924}
3925
3926Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
3927 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3928 std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
3929 return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
3930}
3931
3932GlobalVariable *
3933OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
3934 std::string VarName) {
3935 llvm::Constant *MaptypesArrayInit =
3936 llvm::ConstantDataArray::get(M.getContext(), Mappings);
3937 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
3938 M, MaptypesArrayInit->getType(),
3939 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
3940 VarName);
3941 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3942 return MaptypesArrayGlobal;
3943}
3944
3945void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
3946 InsertPointTy AllocaIP,
3947 unsigned NumOperands,
3948 struct MapperAllocas &MapperAllocas) {
3949 if (!updateToLocation(Loc))
1
Taking false branch
3950 return;
3951
3952 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
3953 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
3954 Builder.restoreIP(AllocaIP);
2
Calling 'IRBuilderBase::restoreIP'
7
Returning from 'IRBuilderBase::restoreIP'
3955 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI8PtrTy);
8
Calling 'IRBuilderBase::CreateAlloca'
3956 AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy);
3957 AllocaInst *ArgSizes = Builder.CreateAlloca(ArrI64Ty);
3958 Builder.restoreIP(Loc.IP);
3959 MapperAllocas.ArgsBase = ArgsBase;
3960 MapperAllocas.Args = Args;
3961 MapperAllocas.ArgSizes = ArgSizes;
3962}
3963
3964void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
3965 Function *MapperFunc, Value *SrcLocInfo,
3966 Value *MaptypesArg, Value *MapnamesArg,
3967 struct MapperAllocas &MapperAllocas,
3968 int64_t DeviceID, unsigned NumOperands) {
3969 if (!updateToLocation(Loc))
3970 return;
3971
3972 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
3973 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
3974 Value *ArgsBaseGEP =
3975 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
3976 {Builder.getInt32(0), Builder.getInt32(0)});
3977 Value *ArgsGEP =
3978 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
3979 {Builder.getInt32(0), Builder.getInt32(0)});
3980 Value *ArgSizesGEP =
3981 Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
3982 {Builder.getInt32(0), Builder.getInt32(0)});
3983 Value *NullPtr = Constant::getNullValue(Int8Ptr->getPointerTo());
3984 Builder.CreateCall(MapperFunc,
3985 {SrcLocInfo, Builder.getInt64(DeviceID),
3986 Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP,
3987 ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
3988}
3989
3990bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
3991 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
3992 assert(!(AO == AtomicOrdering::NotAtomic ||(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3994, __extension__
__PRETTY_FUNCTION__))
3993 AO == llvm::AtomicOrdering::Unordered) &&(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3994, __extension__
__PRETTY_FUNCTION__))
3994 "Unexpected Atomic Ordering.")(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3994, __extension__
__PRETTY_FUNCTION__))
;
3995
3996 bool Flush = false;
3997 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
3998
3999 switch (AK) {
4000 case Read:
4001 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
4002 AO == AtomicOrdering::SequentiallyConsistent) {
4003 FlushAO = AtomicOrdering::Acquire;
4004 Flush = true;
4005 }
4006 break;
4007 case Write:
4008 case Compare:
4009 case Update:
4010 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
4011 AO == AtomicOrdering::SequentiallyConsistent) {
4012 FlushAO = AtomicOrdering::Release;
4013 Flush = true;
4014 }
4015 break;
4016 case Capture:
4017 switch (AO) {
4018 case AtomicOrdering::Acquire:
4019 FlushAO = AtomicOrdering::Acquire;
4020 Flush = true;
4021 break;
4022 case AtomicOrdering::Release:
4023 FlushAO = AtomicOrdering::Release;
4024 Flush = true;
4025 break;
4026 case AtomicOrdering::AcquireRelease:
4027 case AtomicOrdering::SequentiallyConsistent:
4028 FlushAO = AtomicOrdering::AcquireRelease;
4029 Flush = true;
4030 break;
4031 default:
4032 // do nothing - leave silently.
4033 break;
4034 }
4035 }
4036
4037 if (Flush) {
4038 // Currently Flush RT call still doesn't take memory_ordering, so for when
4039 // that happens, this tries to do the resolution of which atomic ordering
4040 // to use with but issue the flush call
4041 // TODO: pass `FlushAO` after memory ordering support is added
4042 (void)FlushAO;
4043 emitFlush(Loc);
4044 }
4045
4046 // for AO == AtomicOrdering::Monotonic and all other case combinations
4047 // do nothing
4048 return Flush;
4049}
4050
4051OpenMPIRBuilder::InsertPointTy
4052OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
4053 AtomicOpValue &X, AtomicOpValue &V,
4054 AtomicOrdering AO) {
4055 if (!updateToLocation(Loc))
4056 return Loc.IP;
4057
4058 Type *XTy = X.Var->getType();
4059 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4059, __extension__
__PRETTY_FUNCTION__))
;
4060 Type *XElemTy = X.ElemTy;
4061 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4063, __extension__
__PRETTY_FUNCTION__))
4062 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4063, __extension__
__PRETTY_FUNCTION__))
4063 "OMP atomic read expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4063, __extension__
__PRETTY_FUNCTION__))
;
4064
4065 Value *XRead = nullptr;
4066
4067 if (XElemTy->isIntegerTy()) {
4068 LoadInst *XLD =
4069 Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");
4070 XLD->setAtomic(AO);
4071 XRead = cast<Value>(XLD);
4072 } else {
4073 // We need to bitcast and perform atomic op as integer
4074 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
4075 IntegerType *IntCastTy =
4076 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4077 Value *XBCast = Builder.CreateBitCast(
4078 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.src.int.cast");
4079 LoadInst *XLoad =
4080 Builder.CreateLoad(IntCastTy, XBCast, X.IsVolatile, "omp.atomic.load");
4081 XLoad->setAtomic(AO);
4082 if (XElemTy->isFloatingPointTy()) {
4083 XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");
4084 } else {
4085 XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");
4086 }
4087 }
4088 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
4089 Builder.CreateStore(XRead, V.Var, V.IsVolatile);
4090 return Builder.saveIP();
4091}
4092
4093OpenMPIRBuilder::InsertPointTy
4094OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
4095 AtomicOpValue &X, Value *Expr,
4096 AtomicOrdering AO) {
4097 if (!updateToLocation(Loc))
4098 return Loc.IP;
4099
4100 Type *XTy = X.Var->getType();
4101 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__))
;
4102 Type *XElemTy = X.ElemTy;
4103 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4105, __extension__
__PRETTY_FUNCTION__))
4104 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4105, __extension__
__PRETTY_FUNCTION__))
4105 "OMP atomic write expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4105, __extension__
__PRETTY_FUNCTION__))
;
4106
4107 if (XElemTy->isIntegerTy()) {
4108 StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);
4109 XSt->setAtomic(AO);
4110 } else {
4111 // We need to bitcast and perform atomic op as integers
4112 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
4113 IntegerType *IntCastTy =
4114 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4115 Value *XBCast = Builder.CreateBitCast(
4116 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.dst.int.cast");
4117 Value *ExprCast =
4118 Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");
4119 StoreInst *XSt = Builder.CreateStore(ExprCast, XBCast, X.IsVolatile);
4120 XSt->setAtomic(AO);
4121 }
4122
4123 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
4124 return Builder.saveIP();
4125}
4126
4127OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
4128 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
4129 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
4130 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
4131 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous")(static_cast <bool> (!isConflictIP(Loc.IP, AllocaIP) &&
"IPs must not be ambiguous") ? void (0) : __assert_fail ("!isConflictIP(Loc.IP, AllocaIP) && \"IPs must not be ambiguous\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4131, __extension__
__PRETTY_FUNCTION__))
;
4132 if (!updateToLocation(Loc))
4133 return Loc.IP;
4134
4135 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4136 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4137 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4138 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4139 Type *XElemTy = X.ElemTy;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4140 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4141 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4142 "OMP atomic update expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4143 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4144 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4145 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4146 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4142, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4145, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
4147
4148 emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
4149 X.IsVolatile, IsXBinopExpr);
4150 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
4151 return Builder.saveIP();
4152}
4153
4154Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
4155 AtomicRMWInst::BinOp RMWOp) {
4156 switch (RMWOp) {
4157 case AtomicRMWInst::Add:
4158 return Builder.CreateAdd(Src1, Src2);
4159 case AtomicRMWInst::Sub:
4160 return Builder.CreateSub(Src1, Src2);
4161 case AtomicRMWInst::And:
4162 return Builder.CreateAnd(Src1, Src2);
4163 case AtomicRMWInst::Nand:
4164 return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));
4165 case AtomicRMWInst::Or:
4166 return Builder.CreateOr(Src1, Src2);
4167 case AtomicRMWInst::Xor:
4168 return Builder.CreateXor(Src1, Src2);
4169 case AtomicRMWInst::Xchg:
4170 case AtomicRMWInst::FAdd:
4171 case AtomicRMWInst::FSub:
4172 case AtomicRMWInst::BAD_BINOP:
4173 case AtomicRMWInst::Max:
4174 case AtomicRMWInst::Min:
4175 case AtomicRMWInst::UMax:
4176 case AtomicRMWInst::UMin:
4177 case AtomicRMWInst::FMax:
4178 case AtomicRMWInst::FMin:
4179 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4179)
;
4180 }
4181 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4181)
;
4182}
4183
4184std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
4185 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
4186 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
4187 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
4188 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
4189 // or a complex datatype.
4190 bool emitRMWOp = false;
4191 switch (RMWOp) {
4192 case AtomicRMWInst::Add:
4193 case AtomicRMWInst::And:
4194 case AtomicRMWInst::Nand:
4195 case AtomicRMWInst::Or:
4196 case AtomicRMWInst::Xor:
4197 case AtomicRMWInst::Xchg:
4198 emitRMWOp = XElemTy;
4199 break;
4200 case AtomicRMWInst::Sub:
4201 emitRMWOp = (IsXBinopExpr && XElemTy);
4202 break;
4203 default:
4204 emitRMWOp = false;
4205 }
4206 emitRMWOp &= XElemTy->isIntegerTy();
4207
4208 std::pair<Value *, Value *> Res;
4209 if (emitRMWOp) {
4210 Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
4211 // not needed except in case of postfix captures. Generate anyway for
4212 // consistency with the else part. Will be removed with any DCE pass.
4213 // AtomicRMWInst::Xchg does not have a coressponding instruction.
4214 if (RMWOp == AtomicRMWInst::Xchg)
4215 Res.second = Res.first;
4216 else
4217 Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
4218 } else {
4219 unsigned Addrspace = cast<PointerType>(X->getType())->getAddressSpace();
4220 IntegerType *IntCastTy =
4221 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4222 Value *XBCast =
4223 Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
4224 LoadInst *OldVal =
4225 Builder.CreateLoad(IntCastTy, XBCast, X->getName() + ".atomic.load");
4226 OldVal->setAtomic(AO);
4227 // CurBB
4228 // | /---\
4229 // ContBB |
4230 // | \---/
4231 // ExitBB
4232 BasicBlock *CurBB = Builder.GetInsertBlock();
4233 Instruction *CurBBTI = CurBB->getTerminator();
4234 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
4235 BasicBlock *ExitBB =
4236 CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
4237 BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
4238 X->getName() + ".atomic.cont");
4239 ContBB->getTerminator()->eraseFromParent();
4240 Builder.restoreIP(AllocaIP);
4241 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
4242 NewAtomicAddr->setName(X->getName() + "x.new.val");
4243 Builder.SetInsertPoint(ContBB);
4244 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
4245 PHI->addIncoming(OldVal, CurBB);
4246 IntegerType *NewAtomicCastTy =
4247 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4248 bool IsIntTy = XElemTy->isIntegerTy();
4249 Value *NewAtomicIntAddr =
4250 (IsIntTy)
4251 ? NewAtomicAddr
4252 : Builder.CreateBitCast(NewAtomicAddr,
4253 NewAtomicCastTy->getPointerTo(Addrspace));
4254 Value *OldExprVal = PHI;
4255 if (!IsIntTy) {
4256 if (XElemTy->isFloatingPointTy()) {
4257 OldExprVal = Builder.CreateBitCast(PHI, XElemTy,
4258 X->getName() + ".atomic.fltCast");
4259 } else {
4260 OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,
4261 X->getName() + ".atomic.ptrCast");
4262 }
4263 }
4264
4265 Value *Upd = UpdateOp(OldExprVal, Builder);
4266 Builder.CreateStore(Upd, NewAtomicAddr);
4267 LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicIntAddr);
4268 Value *XAddr =
4269 (IsIntTy)
4270 ? X
4271 : Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
4272 AtomicOrdering Failure =
4273 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
4274 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
4275 XAddr, PHI, DesiredVal, llvm::MaybeAlign(), AO, Failure);
4276 Result->setVolatile(VolatileX);
4277 Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);
4278 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4279 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());
4280 Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);
4281
4282 Res.first = OldExprVal;
4283 Res.second = Upd;
4284
4285 // set Insertion point in exit block
4286 if (UnreachableInst *ExitTI =
4287 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
4288 CurBBTI->eraseFromParent();
4289 Builder.SetInsertPoint(ExitBB);
4290 } else {
4291 Builder.SetInsertPoint(ExitTI);
4292 }
4293 }
4294
4295 return Res;
4296}
4297
4298OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
4299 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
4300 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
4301 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
4302 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
4303 if (!updateToLocation(Loc))
4304 return Loc.IP;
4305
4306 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4307 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4308 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4309 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4310 Type *XElemTy = X.ElemTy;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4311 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4312 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4313 "OMP atomic capture expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4314 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4315 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4316 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4309, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4313, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4315, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
4317
4318 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
4319 // 'x' is simply atomically rewritten with 'expr'.
4320 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
4321 std::pair<Value *, Value *> Result =
4322 emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
4323 X.IsVolatile, IsXBinopExpr);
4324
4325 Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
4326 Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
4327
4328 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
4329 return Builder.saveIP();
4330}
4331
4332OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
4333 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
4334 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
4335 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
4336 bool IsFailOnly) {
4337
4338 if (!updateToLocation(Loc))
4339 return Loc.IP;
4340
4341 assert(X.Var->getType()->isPointerTy() &&(static_cast <bool> (X.Var->getType()->isPointerTy
() && "OMP atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("X.Var->getType()->isPointerTy() && \"OMP atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4342, __extension__
__PRETTY_FUNCTION__))
4342 "OMP atomic expects a pointer to target memory")(static_cast <bool> (X.Var->getType()->isPointerTy
() && "OMP atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("X.Var->getType()->isPointerTy() && \"OMP atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4342, __extension__
__PRETTY_FUNCTION__))
;
4343 // compare capture
4344 if (V.Var) {
4345 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type")(static_cast <bool> (V.Var->getType()->isPointerTy
() && "v.var must be of pointer type") ? void (0) : __assert_fail
("V.Var->getType()->isPointerTy() && \"v.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4345, __extension__
__PRETTY_FUNCTION__))
;
4346 assert(V.ElemTy == X.ElemTy && "x and v must be of same type")(static_cast <bool> (V.ElemTy == X.ElemTy && "x and v must be of same type"
) ? void (0) : __assert_fail ("V.ElemTy == X.ElemTy && \"x and v must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4346, __extension__
__PRETTY_FUNCTION__))
;
4347 }
4348
4349 bool IsInteger = E->getType()->isIntegerTy();
4350
4351 if (Op == OMPAtomicCompareOp::EQ) {
4352 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
4353 AtomicCmpXchgInst *Result = nullptr;
4354 if (!IsInteger) {
4355 unsigned Addrspace =
4356 cast<PointerType>(X.Var->getType())->getAddressSpace();
4357 IntegerType *IntCastTy =
4358 IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits());
4359 Value *XBCast =
4360 Builder.CreateBitCast(X.Var, IntCastTy->getPointerTo(Addrspace));
4361 Value *EBCast = Builder.CreateBitCast(E, IntCastTy);
4362 Value *DBCast = Builder.CreateBitCast(D, IntCastTy);
4363 Result = Builder.CreateAtomicCmpXchg(XBCast, EBCast, DBCast, MaybeAlign(),
4364 AO, Failure);
4365 } else {
4366 Result =
4367 Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);
4368 }
4369
4370 if (V.Var) {
4371 Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0);
4372 if (!IsInteger)
4373 OldValue = Builder.CreateBitCast(OldValue, X.ElemTy);
4374 assert(OldValue->getType() == V.ElemTy &&(static_cast <bool> (OldValue->getType() == V.ElemTy
&& "OldValue and V must be of same type") ? void (0)
: __assert_fail ("OldValue->getType() == V.ElemTy && \"OldValue and V must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4375, __extension__
__PRETTY_FUNCTION__))
4375 "OldValue and V must be of same type")(static_cast <bool> (OldValue->getType() == V.ElemTy
&& "OldValue and V must be of same type") ? void (0)
: __assert_fail ("OldValue->getType() == V.ElemTy && \"OldValue and V must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4375, __extension__
__PRETTY_FUNCTION__))
;
4376 if (IsPostfixUpdate) {
4377 Builder.CreateStore(OldValue, V.Var, V.IsVolatile);
4378 } else {
4379 Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4380 if (IsFailOnly) {
4381 // CurBB----
4382 // | |
4383 // v |
4384 // ContBB |
4385 // | |
4386 // v |
4387 // ExitBB <-
4388 //
4389 // where ContBB only contains the store of old value to 'v'.
4390 BasicBlock *CurBB = Builder.GetInsertBlock();
4391 Instruction *CurBBTI = CurBB->getTerminator();
4392 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
4393 BasicBlock *ExitBB = CurBB->splitBasicBlock(
4394 CurBBTI, X.Var->getName() + ".atomic.exit");
4395 BasicBlock *ContBB = CurBB->splitBasicBlock(
4396 CurBB->getTerminator(), X.Var->getName() + ".atomic.cont");
4397 ContBB->getTerminator()->eraseFromParent();
4398 CurBB->getTerminator()->eraseFromParent();
4399
4400 Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB);
4401
4402 Builder.SetInsertPoint(ContBB);
4403 Builder.CreateStore(OldValue, V.Var);
4404 Builder.CreateBr(ExitBB);
4405
4406 if (UnreachableInst *ExitTI =
4407 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
4408 CurBBTI->eraseFromParent();
4409 Builder.SetInsertPoint(ExitBB);
4410 } else {
4411 Builder.SetInsertPoint(ExitTI);
4412 }
4413 } else {
4414 Value *CapturedValue =
4415 Builder.CreateSelect(SuccessOrFail, E, OldValue);
4416 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
4417 }
4418 }
4419 }
4420 // The comparison result has to be stored.
4421 if (R.Var) {
4422 assert(R.Var->getType()->isPointerTy() &&(static_cast <bool> (R.Var->getType()->isPointerTy
() && "r.var must be of pointer type") ? void (0) : __assert_fail
("R.Var->getType()->isPointerTy() && \"r.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4423, __extension__
__PRETTY_FUNCTION__))
4423 "r.var must be of pointer type")(static_cast <bool> (R.Var->getType()->isPointerTy
() && "r.var must be of pointer type") ? void (0) : __assert_fail
("R.Var->getType()->isPointerTy() && \"r.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4423, __extension__
__PRETTY_FUNCTION__))
;
4424 assert(R.ElemTy->isIntegerTy() && "r must be of integral type")(static_cast <bool> (R.ElemTy->isIntegerTy() &&
"r must be of integral type") ? void (0) : __assert_fail ("R.ElemTy->isIntegerTy() && \"r must be of integral type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4424, __extension__
__PRETTY_FUNCTION__))
;
4425
4426 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4427 Value *ResultCast = R.IsSigned
4428 ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy)
4429 : Builder.CreateZExt(SuccessFailureVal, R.ElemTy);
4430 Builder.CreateStore(ResultCast, R.Var, R.IsVolatile);
4431 }
4432 } else {
4433 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&(static_cast <bool> ((Op == OMPAtomicCompareOp::MAX || Op
== OMPAtomicCompareOp::MIN) && "Op should be either max or min at this point"
) ? void (0) : __assert_fail ("(Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) && \"Op should be either max or min at this point\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4434, __extension__
__PRETTY_FUNCTION__))
4434 "Op should be either max or min at this point")(static_cast <bool> ((Op == OMPAtomicCompareOp::MAX || Op
== OMPAtomicCompareOp::MIN) && "Op should be either max or min at this point"
) ? void (0) : __assert_fail ("(Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) && \"Op should be either max or min at this point\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4434, __extension__
__PRETTY_FUNCTION__))
;
4435 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==")(static_cast <bool> (!IsFailOnly && "IsFailOnly is only valid when the comparison is =="
) ? void (0) : __assert_fail ("!IsFailOnly && \"IsFailOnly is only valid when the comparison is ==\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4435, __extension__
__PRETTY_FUNCTION__))
;
4436
4437 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
4438 // Let's take max as example.
4439 // OpenMP form:
4440 // x = x > expr ? expr : x;
4441 // LLVM form:
4442 // *ptr = *ptr > val ? *ptr : val;
4443 // We need to transform to LLVM form.
4444 // x = x <= expr ? x : expr;
4445 AtomicRMWInst::BinOp NewOp;
4446 if (IsXBinopExpr) {
4447 if (IsInteger) {
4448 if (X.IsSigned)
4449 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
4450 : AtomicRMWInst::Max;
4451 else
4452 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
4453 : AtomicRMWInst::UMax;
4454 } else {
4455 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
4456 : AtomicRMWInst::FMax;
4457 }
4458 } else {
4459 if (IsInteger) {
4460 if (X.IsSigned)
4461 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
4462 : AtomicRMWInst::Min;
4463 else
4464 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
4465 : AtomicRMWInst::UMin;
4466 } else {
4467 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
4468 : AtomicRMWInst::FMin;
4469 }
4470 }
4471
4472 AtomicRMWInst *OldValue =
4473 Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);
4474 if (V.Var) {
4475 Value *CapturedValue = nullptr;
4476 if (IsPostfixUpdate) {
4477 CapturedValue = OldValue;
4478 } else {
4479 CmpInst::Predicate Pred;
4480 switch (NewOp) {
4481 case AtomicRMWInst::Max:
4482 Pred = CmpInst::ICMP_SGT;
4483 break;
4484 case AtomicRMWInst::UMax:
4485 Pred = CmpInst::ICMP_UGT;
4486 break;
4487 case AtomicRMWInst::FMax:
4488 Pred = CmpInst::FCMP_OGT;
4489 break;
4490 case AtomicRMWInst::Min:
4491 Pred = CmpInst::ICMP_SLT;
4492 break;
4493 case AtomicRMWInst::UMin:
4494 Pred = CmpInst::ICMP_ULT;
4495 break;
4496 case AtomicRMWInst::FMin:
4497 Pred = CmpInst::FCMP_OLT;
4498 break;
4499 default:
4500 llvm_unreachable("unexpected comparison op")::llvm::llvm_unreachable_internal("unexpected comparison op",
"llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4500)
;
4501 }
4502 Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E);
4503 CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue);
4504 }
4505 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
4506 }
4507 }
4508
4509 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);
4510
4511 return Builder.saveIP();
4512}
4513
4514GlobalVariable *
4515OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
4516 std::string VarName) {
4517 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
4518 llvm::ArrayType::get(
4519 llvm::Type::getInt8Ty(M.getContext())->getPointerTo(), Names.size()),
4520 Names);
4521 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
4522 M, MapNamesArrayInit->getType(),
4523 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
4524 VarName);
4525 return MapNamesArrayGlobal;
4526}
4527
4528// Create all simple and struct types exposed by the runtime and remember
4529// the llvm::PointerTypes of them for easy access later.
4530void OpenMPIRBuilder::initializeTypes(Module &M) {
4531 LLVMContext &Ctx = M.getContext();
4532 StructType *T;
4533#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
4534#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
4535 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
4536 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
4537#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
4538 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
4539 VarName##Ptr = PointerType::getUnqual(VarName);
4540#define OMP_STRUCT_TYPE(VarName, StructName, ...) \
4541 T = StructType::getTypeByName(Ctx, StructName); \
4542 if (!T) \
4543 T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
4544 VarName = T; \
4545 VarName##Ptr = PointerType::getUnqual(T);
4546#include "llvm/Frontend/OpenMP/OMPKinds.def"
4547}
4548
4549void OpenMPIRBuilder::OutlineInfo::collectBlocks(
4550 SmallPtrSetImpl<BasicBlock *> &BlockSet,
4551 SmallVectorImpl<BasicBlock *> &BlockVector) {
4552 SmallVector<BasicBlock *, 32> Worklist;
4553 BlockSet.insert(EntryBB);
4554 BlockSet.insert(ExitBB);
4555
4556 Worklist.push_back(EntryBB);
4557 while (!Worklist.empty()) {
4558 BasicBlock *BB = Worklist.pop_back_val();
4559 BlockVector.push_back(BB);
4560 for (BasicBlock *SuccBB : successors(BB))
4561 if (BlockSet.insert(SuccBB).second)
4562 Worklist.push_back(SuccBB);
4563 }
4564}
4565
4566void CanonicalLoopInfo::collectControlBlocks(
4567 SmallVectorImpl<BasicBlock *> &BBs) {
4568 // We only count those BBs as control block for which we do not need to
4569 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
4570 // flow. For consistency, this also means we do not add the Body block, which
4571 // is just the entry to the body code.
4572 BBs.reserve(BBs.size() + 6);
4573 BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});
4574}
4575
4576BasicBlock *CanonicalLoopInfo::getPreheader() const {
4577 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4577, __extension__
__PRETTY_FUNCTION__))
;
4578 for (BasicBlock *Pred : predecessors(Header)) {
4579 if (Pred != Latch)
4580 return Pred;
4581 }
4582 llvm_unreachable("Missing preheader")::llvm::llvm_unreachable_internal("Missing preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 4582)
;
4583}
4584
4585void CanonicalLoopInfo::setTripCount(Value *TripCount) {
4586 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4586, __extension__
__PRETTY_FUNCTION__))
;
4587
4588 Instruction *CmpI = &getCond()->front();
4589 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount")(static_cast <bool> (isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount") ? void (0) : __assert_fail
("isa<CmpInst>(CmpI) && \"First inst must compare IV with TripCount\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4589, __extension__
__PRETTY_FUNCTION__))
;
4590 CmpI->setOperand(1, TripCount);
4591
4592#ifndef NDEBUG
4593 assertOK();
4594#endif
4595}
4596
4597void CanonicalLoopInfo::mapIndVar(
4598 llvm::function_ref<Value *(Instruction *)> Updater) {
4599 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4599, __extension__
__PRETTY_FUNCTION__))
;
4600
4601 Instruction *OldIV = getIndVar();
4602
4603 // Record all uses excluding those introduced by the updater. Uses by the
4604 // CanonicalLoopInfo itself to keep track of the number of iterations are
4605 // excluded.
4606 SmallVector<Use *> ReplacableUses;
4607 for (Use &U : OldIV->uses()) {
4608 auto *User = dyn_cast<Instruction>(U.getUser());
4609 if (!User)
4610 continue;
4611 if (User->getParent() == getCond())
4612 continue;
4613 if (User->getParent() == getLatch())
4614 continue;
4615 ReplacableUses.push_back(&U);
4616 }
4617
4618 // Run the updater that may introduce new uses
4619 Value *NewIV = Updater(OldIV);
4620
4621 // Replace the old uses with the value returned by the updater.
4622 for (Use *U : ReplacableUses)
4623 U->set(NewIV);
4624
4625#ifndef NDEBUG
4626 assertOK();
4627#endif
4628}
4629
4630void CanonicalLoopInfo::assertOK() const {
4631#ifndef NDEBUG
4632 // No constraints if this object currently does not describe a loop.
4633 if (!isValid())
4634 return;
4635
4636 BasicBlock *Preheader = getPreheader();
4637 BasicBlock *Body = getBody();
4638 BasicBlock *After = getAfter();
4639
4640 // Verify standard control-flow we use for OpenMP loops.
4641 assert(Preheader)(static_cast <bool> (Preheader) ? void (0) : __assert_fail
("Preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4641
, __extension__ __PRETTY_FUNCTION__))
;
4642 assert(isa<BranchInst>(Preheader->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4643, __extension__
__PRETTY_FUNCTION__))
4643 "Preheader must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4643, __extension__
__PRETTY_FUNCTION__))
;
4644 assert(Preheader->getSingleSuccessor() == Header &&(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4645, __extension__
__PRETTY_FUNCTION__))
4645 "Preheader must jump to header")(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4645, __extension__
__PRETTY_FUNCTION__))
;
4646
4647 assert(Header)(static_cast <bool> (Header) ? void (0) : __assert_fail
("Header", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4647
, __extension__ __PRETTY_FUNCTION__))
;
4648 assert(isa<BranchInst>(Header->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4649, __extension__
__PRETTY_FUNCTION__))
4649 "Header must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4649, __extension__
__PRETTY_FUNCTION__))
;
4650 assert(Header->getSingleSuccessor() == Cond &&(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4651, __extension__
__PRETTY_FUNCTION__))
4651 "Header must jump to exiting block")(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4651, __extension__
__PRETTY_FUNCTION__))
;
4652
4653 assert(Cond)(static_cast <bool> (Cond) ? void (0) : __assert_fail (
"Cond", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4653, __extension__
__PRETTY_FUNCTION__))
;
4654 assert(Cond->getSinglePredecessor() == Header &&(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4655, __extension__
__PRETTY_FUNCTION__))
4655 "Exiting block only reachable from header")(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4655, __extension__
__PRETTY_FUNCTION__))
;
4656
4657 assert(isa<BranchInst>(Cond->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4658, __extension__
__PRETTY_FUNCTION__))
4658 "Exiting block must terminate with conditional branch")(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4658, __extension__
__PRETTY_FUNCTION__))
;
4659 assert(size(successors(Cond)) == 2 &&(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4660, __extension__
__PRETTY_FUNCTION__))
4660 "Exiting block must have two successors")(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4660, __extension__
__PRETTY_FUNCTION__))
;
4661 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4662, __extension__
__PRETTY_FUNCTION__))
4662 "Exiting block's first successor jump to the body")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4662, __extension__
__PRETTY_FUNCTION__))
;
4663 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4664, __extension__
__PRETTY_FUNCTION__))
4664 "Exiting block's second successor must exit the loop")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4664, __extension__
__PRETTY_FUNCTION__))
;
4665
4666 assert(Body)(static_cast <bool> (Body) ? void (0) : __assert_fail (
"Body", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4666, __extension__
__PRETTY_FUNCTION__))
;
4667 assert(Body->getSinglePredecessor() == Cond &&(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4668, __extension__
__PRETTY_FUNCTION__))
4668 "Body only reachable from exiting block")(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4668, __extension__
__PRETTY_FUNCTION__))
;
4669 assert(!isa<PHINode>(Body->front()))(static_cast <bool> (!isa<PHINode>(Body->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Body->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4669, __extension__
__PRETTY_FUNCTION__))
;
4670
4671 assert(Latch)(static_cast <bool> (Latch) ? void (0) : __assert_fail (
"Latch", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4671, __extension__
__PRETTY_FUNCTION__))
;
4672 assert(isa<BranchInst>(Latch->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4673, __extension__
__PRETTY_FUNCTION__))
4673 "Latch must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4673, __extension__
__PRETTY_FUNCTION__))
;
4674 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header")(static_cast <bool> (Latch->getSingleSuccessor() == Header
&& "Latch must jump to header") ? void (0) : __assert_fail
("Latch->getSingleSuccessor() == Header && \"Latch must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4674, __extension__
__PRETTY_FUNCTION__))
;
4675 // TODO: To support simple redirecting of the end of the body code that has
4676 // multiple; introduce another auxiliary basic block like preheader and after.
4677 assert(Latch->getSinglePredecessor() != nullptr)(static_cast <bool> (Latch->getSinglePredecessor() !=
nullptr) ? void (0) : __assert_fail ("Latch->getSinglePredecessor() != nullptr"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4677, __extension__
__PRETTY_FUNCTION__))
;
4678 assert(!isa<PHINode>(Latch->front()))(static_cast <bool> (!isa<PHINode>(Latch->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Latch->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4678, __extension__
__PRETTY_FUNCTION__))
;
4679
4680 assert(Exit)(static_cast <bool> (Exit) ? void (0) : __assert_fail (
"Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4680, __extension__
__PRETTY_FUNCTION__))
;
4681 assert(isa<BranchInst>(Exit->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4682, __extension__
__PRETTY_FUNCTION__))
4682 "Exit block must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4682, __extension__
__PRETTY_FUNCTION__))
;
4683 assert(Exit->getSingleSuccessor() == After &&(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4684, __extension__
__PRETTY_FUNCTION__))
4684 "Exit block must jump to after block")(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4684, __extension__
__PRETTY_FUNCTION__))
;
4685
4686 assert(After)(static_cast <bool> (After) ? void (0) : __assert_fail (
"After", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4686, __extension__
__PRETTY_FUNCTION__))
;
4687 assert(After->getSinglePredecessor() == Exit &&(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4688, __extension__
__PRETTY_FUNCTION__))
4688 "After block only reachable from exit block")(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4688, __extension__
__PRETTY_FUNCTION__))
;
4689 assert(After->empty() || !isa<PHINode>(After->front()))(static_cast <bool> (After->empty() || !isa<PHINode
>(After->front())) ? void (0) : __assert_fail ("After->empty() || !isa<PHINode>(After->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4689, __extension__
__PRETTY_FUNCTION__))
;
4690
4691 Instruction *IndVar = getIndVar();
4692 assert(IndVar && "Canonical induction variable not found?")(static_cast <bool> (IndVar && "Canonical induction variable not found?"
) ? void (0) : __assert_fail ("IndVar && \"Canonical induction variable not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4692, __extension__
__PRETTY_FUNCTION__))
;
4693 assert(isa<IntegerType>(IndVar->getType()) &&(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4694, __extension__
__PRETTY_FUNCTION__))
4694 "Induction variable must be an integer")(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4694, __extension__
__PRETTY_FUNCTION__))
;
4695 assert(cast<PHINode>(IndVar)->getParent() == Header &&(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4696, __extension__
__PRETTY_FUNCTION__))
4696 "Induction variable must be a PHI in the loop header")(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4696, __extension__
__PRETTY_FUNCTION__))
;
4697 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(0) == Preheader) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4697, __extension__
__PRETTY_FUNCTION__))
;
4698 assert((static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4699, __extension__
__PRETTY_FUNCTION__))
4699 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero())(static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4699, __extension__
__PRETTY_FUNCTION__))
;
4700 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(1) == Latch) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4700, __extension__
__PRETTY_FUNCTION__))
;
4701
4702 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
4703 assert(cast<Instruction>(NextIndVar)->getParent() == Latch)(static_cast <bool> (cast<Instruction>(NextIndVar
)->getParent() == Latch) ? void (0) : __assert_fail ("cast<Instruction>(NextIndVar)->getParent() == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4703, __extension__
__PRETTY_FUNCTION__))
;
4704 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOpcode() == BinaryOperator::Add) ? void (0) : __assert_fail
("cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4704, __extension__
__PRETTY_FUNCTION__))
;
4705 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOperand(0) == IndVar) ? void (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4705, __extension__
__PRETTY_FUNCTION__))
;
4706 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4707, __extension__
__PRETTY_FUNCTION__))
4707 ->isOne())(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4707, __extension__
__PRETTY_FUNCTION__))
;
4708
4709 Value *TripCount = getTripCount();
4710 assert(TripCount && "Loop trip count not found?")(static_cast <bool> (TripCount && "Loop trip count not found?"
) ? void (0) : __assert_fail ("TripCount && \"Loop trip count not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4710, __extension__
__PRETTY_FUNCTION__))
;
4711 assert(IndVar->getType() == TripCount->getType() &&(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4712, __extension__
__PRETTY_FUNCTION__))
4712 "Trip count and induction variable must have the same type")(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4712, __extension__
__PRETTY_FUNCTION__))
;
4713
4714 auto *CmpI = cast<CmpInst>(&Cond->front());
4715 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4716, __extension__
__PRETTY_FUNCTION__))
4716 "Exit condition must be a signed less-than comparison")(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4716, __extension__
__PRETTY_FUNCTION__))
;
4717 assert(CmpI->getOperand(0) == IndVar &&(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4718, __extension__
__PRETTY_FUNCTION__))
4718 "Exit condition must compare the induction variable")(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4718, __extension__
__PRETTY_FUNCTION__))
;
4719 assert(CmpI->getOperand(1) == TripCount &&(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4720, __extension__
__PRETTY_FUNCTION__))
4720 "Exit condition must compare with the trip count")(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4720, __extension__
__PRETTY_FUNCTION__))
;
4721#endif
4722}
4723
4724void CanonicalLoopInfo::invalidate() {
4725 Header = nullptr;
4726 Cond = nullptr;
4727 Latch = nullptr;
4728 Exit = nullptr;
4729}

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugLoc.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/FPEnv.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstdint>
48#include <functional>
49#include <utility>
50
51namespace llvm {
52
53class APInt;
54class Use;
55
56/// This provides the default implementation of the IRBuilder
57/// 'InsertHelper' method that is called whenever an instruction is created by
58/// IRBuilder and needs to be inserted.
59///
60/// By default, this inserts the instruction at the insertion point.
61class IRBuilderDefaultInserter {
62public:
63 virtual ~IRBuilderDefaultInserter();
64
65 virtual void InsertHelper(Instruction *I, const Twine &Name,
66 BasicBlock *BB,
67 BasicBlock::iterator InsertPt) const {
68 if (BB) BB->getInstList().insert(InsertPt, I);
69 I->setName(Name);
70 }
71};
72
73/// Provides an 'InsertHelper' that calls a user-provided callback after
74/// performing the default insertion.
75class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
76 std::function<void(Instruction *)> Callback;
77
78public:
79 ~IRBuilderCallbackInserter() override;
80
81 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
82 : Callback(std::move(Callback)) {}
83
84 void InsertHelper(Instruction *I, const Twine &Name,
85 BasicBlock *BB,
86 BasicBlock::iterator InsertPt) const override {
87 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
88 Callback(I);
89 }
90};
91
92/// Common base class shared among various IRBuilders.
93class IRBuilderBase {
94 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
95 /// created instructions, like !dbg metadata.
96 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
97
98 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
99 /// null. If \p MD is null, remove the entry with \p Kind.
100 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
101 if (!MD) {
102 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
103 return KV.first == Kind;
104 });
105 return;
106 }
107
108 for (auto &KV : MetadataToCopy)
109 if (KV.first == Kind) {
110 KV.second = MD;
111 return;
112 }
113
114 MetadataToCopy.emplace_back(Kind, MD);
115 }
116
117protected:
118 BasicBlock *BB;
119 BasicBlock::iterator InsertPt;
120 LLVMContext &Context;
121 const IRBuilderFolder &Folder;
122 const IRBuilderDefaultInserter &Inserter;
123
124 MDNode *DefaultFPMathTag;
125 FastMathFlags FMF;
126
127 bool IsFPConstrained = false;
128 fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebStrict;
129 RoundingMode DefaultConstrainedRounding = RoundingMode::Dynamic;
130
131 ArrayRef<OperandBundleDef> DefaultOperandBundles;
132
133public:
134 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
135 const IRBuilderDefaultInserter &Inserter, MDNode *FPMathTag,
136 ArrayRef<OperandBundleDef> OpBundles)
137 : Context(context), Folder(Folder), Inserter(Inserter),
138 DefaultFPMathTag(FPMathTag), DefaultOperandBundles(OpBundles) {
139 ClearInsertionPoint();
140 }
141
142 /// Insert and return the specified instruction.
143 template<typename InstTy>
144 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
145 Inserter.InsertHelper(I, Name, BB, InsertPt);
146 AddMetadataToInst(I);
147 return I;
148 }
149
150 /// No-op overload to handle constants.
151 Constant *Insert(Constant *C, const Twine& = "") const {
152 return C;
153 }
154
155 Value *Insert(Value *V, const Twine &Name = "") const {
156 if (Instruction *I = dyn_cast<Instruction>(V))
157 return Insert(I, Name);
158 assert(isa<Constant>(V))(static_cast <bool> (isa<Constant>(V)) ? void (0)
: __assert_fail ("isa<Constant>(V)", "llvm/include/llvm/IR/IRBuilder.h"
, 158, __extension__ __PRETTY_FUNCTION__))
;
159 return V;
160 }
161
162 //===--------------------------------------------------------------------===//
163 // Builder configuration methods
164 //===--------------------------------------------------------------------===//
165
166 /// Clear the insertion point: created instructions will not be
167 /// inserted into a block.
168 void ClearInsertionPoint() {
169 BB = nullptr;
5
Null pointer value stored to field 'BB'
170 InsertPt = BasicBlock::iterator();
171 }
172
173 BasicBlock *GetInsertBlock() const { return BB; }
174 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
175 LLVMContext &getContext() const { return Context; }
176
177 /// This specifies that created instructions should be appended to the
178 /// end of the specified block.
179 void SetInsertPoint(BasicBlock *TheBB) {
180 BB = TheBB;
181 InsertPt = BB->end();
182 }
183
184 /// This specifies that created instructions should be inserted before
185 /// the specified instruction.
186 void SetInsertPoint(Instruction *I) {
187 BB = I->getParent();
188 InsertPt = I->getIterator();
189 assert(InsertPt != BB->end() && "Can't read debug loc from end()")(static_cast <bool> (InsertPt != BB->end() &&
"Can't read debug loc from end()") ? void (0) : __assert_fail
("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "llvm/include/llvm/IR/IRBuilder.h", 189, __extension__ __PRETTY_FUNCTION__
))
;
190 SetCurrentDebugLocation(I->getDebugLoc());
191 }
192
193 /// This specifies that created instructions should be inserted at the
194 /// specified point.
195 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
196 BB = TheBB;
197 InsertPt = IP;
198 if (IP != TheBB->end())
199 SetCurrentDebugLocation(IP->getDebugLoc());
200 }
201
202 /// This specifies that created instructions should inserted at the beginning
203 /// end of the specified function, but after already existing static alloca
204 /// instructions that are at the start.
205 void SetInsertPointPastAllocas(Function *F) {
206 BB = &F->getEntryBlock();
207 InsertPt = BB->getFirstNonPHIOrDbgOrAlloca();
208 }
209
210 /// Set location information used by debugging information.
211 void SetCurrentDebugLocation(DebugLoc L) {
212 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
213 }
214
215 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
216 /// added to all created instructions. Entries present in MedataDataToCopy but
217 /// not on \p Src will be dropped from MetadataToCopy.
218 void CollectMetadataToCopy(Instruction *Src,
219 ArrayRef<unsigned> MetadataKinds) {
220 for (unsigned K : MetadataKinds)
221 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
222 }
223
224 /// Get location information used by debugging information.
225 DebugLoc getCurrentDebugLocation() const;
226
227 /// If this builder has a current debug location, set it on the
228 /// specified instruction.
229 void SetInstDebugLocation(Instruction *I) const;
230
231 /// Add all entries in MetadataToCopy to \p I.
232 void AddMetadataToInst(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 I->setMetadata(KV.first, KV.second);
235 }
236
237 /// Get the return type of the current function that we're emitting
238 /// into.
239 Type *getCurrentFunctionReturnType() const;
240
241 /// InsertPoint - A saved insertion point.
242 class InsertPoint {
243 BasicBlock *Block = nullptr;
244 BasicBlock::iterator Point;
245
246 public:
247 /// Creates a new insertion point which doesn't point to anything.
248 InsertPoint() = default;
249
250 /// Creates a new insertion point at the given location.
251 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
252 : Block(InsertBlock), Point(InsertPoint) {}
253
254 /// Returns true if this insert point is set.
255 bool isSet() const { return (Block != nullptr); }
256
257 BasicBlock *getBlock() const { return Block; }
258 BasicBlock::iterator getPoint() const { return Point; }
259 };
260
261 /// Returns the current insert point.
262 InsertPoint saveIP() const {
263 return InsertPoint(GetInsertBlock(), GetInsertPoint());
264 }
265
266 /// Returns the current insert point, clearing it in the process.
267 InsertPoint saveAndClearIP() {
268 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
269 ClearInsertionPoint();
270 return IP;
271 }
272
273 /// Sets the current insert point to a previously-saved location.
274 void restoreIP(InsertPoint IP) {
275 if (IP.isSet())
3
Taking false branch
276 SetInsertPoint(IP.getBlock(), IP.getPoint());
277 else
278 ClearInsertionPoint();
4
Calling 'IRBuilderBase::ClearInsertionPoint'
6
Returning from 'IRBuilderBase::ClearInsertionPoint'
279 }
280
281 /// Get the floating point math metadata being used.
282 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
283
284 /// Get the flags to be applied to created floating point ops
285 FastMathFlags getFastMathFlags() const { return FMF; }
286
287 FastMathFlags &getFastMathFlags() { return FMF; }
288
289 /// Clear the fast-math flags.
290 void clearFastMathFlags() { FMF.clear(); }
291
292 /// Set the floating point math metadata to be used.
293 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
294
295 /// Set the fast-math flags to be used with generated fp-math operators
296 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
297
298 /// Enable/Disable use of constrained floating point math. When
299 /// enabled the CreateF<op>() calls instead create constrained
300 /// floating point intrinsic calls. Fast math flags are unaffected
301 /// by this setting.
302 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
303
304 /// Query for the use of constrained floating point math
305 bool getIsFPConstrained() { return IsFPConstrained; }
306
307 /// Set the exception handling to be used with constrained floating point
308 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
309#ifndef NDEBUG
310 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept);
311 assert(ExceptStr && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 311, __extension__ __PRETTY_FUNCTION__
))
;
312#endif
313 DefaultConstrainedExcept = NewExcept;
314 }
315
316 /// Set the rounding mode handling to be used with constrained floating point
317 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
318#ifndef NDEBUG
319 Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding);
320 assert(RoundingStr && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 320, __extension__ __PRETTY_FUNCTION__
))
;
321#endif
322 DefaultConstrainedRounding = NewRounding;
323 }
324
325 /// Get the exception handling used with constrained floating point
326 fp::ExceptionBehavior getDefaultConstrainedExcept() {
327 return DefaultConstrainedExcept;
328 }
329
330 /// Get the rounding mode handling used with constrained floating point
331 RoundingMode getDefaultConstrainedRounding() {
332 return DefaultConstrainedRounding;
333 }
334
335 void setConstrainedFPFunctionAttr() {
336 assert(BB && "Must have a basic block to set any function attributes!")(static_cast <bool> (BB && "Must have a basic block to set any function attributes!"
) ? void (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "llvm/include/llvm/IR/IRBuilder.h", 336, __extension__ __PRETTY_FUNCTION__
))
;
337
338 Function *F = BB->getParent();
339 if (!F->hasFnAttribute(Attribute::StrictFP)) {
340 F->addFnAttr(Attribute::StrictFP);
341 }
342 }
343
344 void setConstrainedFPCallAttr(CallBase *I) {
345 I->addFnAttr(Attribute::StrictFP);
346 }
347
348 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
349 DefaultOperandBundles = OpBundles;
350 }
351
352 //===--------------------------------------------------------------------===//
353 // RAII helpers.
354 //===--------------------------------------------------------------------===//
355
356 // RAII object that stores the current insertion point and restores it
357 // when the object is destroyed. This includes the debug location.
358 class InsertPointGuard {
359 IRBuilderBase &Builder;
360 AssertingVH<BasicBlock> Block;
361 BasicBlock::iterator Point;
362 DebugLoc DbgLoc;
363
364 public:
365 InsertPointGuard(IRBuilderBase &B)
366 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
367 DbgLoc(B.getCurrentDebugLocation()) {}
368
369 InsertPointGuard(const InsertPointGuard &) = delete;
370 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
371
372 ~InsertPointGuard() {
373 Builder.restoreIP(InsertPoint(Block, Point));
374 Builder.SetCurrentDebugLocation(DbgLoc);
375 }
376 };
377
378 // RAII object that stores the current fast math settings and restores
379 // them when the object is destroyed.
380 class FastMathFlagGuard {
381 IRBuilderBase &Builder;
382 FastMathFlags FMF;
383 MDNode *FPMathTag;
384 bool IsFPConstrained;
385 fp::ExceptionBehavior DefaultConstrainedExcept;
386 RoundingMode DefaultConstrainedRounding;
387
388 public:
389 FastMathFlagGuard(IRBuilderBase &B)
390 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
391 IsFPConstrained(B.IsFPConstrained),
392 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
393 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
394
395 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
396 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
397
398 ~FastMathFlagGuard() {
399 Builder.FMF = FMF;
400 Builder.DefaultFPMathTag = FPMathTag;
401 Builder.IsFPConstrained = IsFPConstrained;
402 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
403 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
404 }
405 };
406
407 // RAII object that stores the current default operand bundles and restores
408 // them when the object is destroyed.
409 class OperandBundlesGuard {
410 IRBuilderBase &Builder;
411 ArrayRef<OperandBundleDef> DefaultOperandBundles;
412
413 public:
414 OperandBundlesGuard(IRBuilderBase &B)
415 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
416
417 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
418 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
419
420 ~OperandBundlesGuard() {
421 Builder.DefaultOperandBundles = DefaultOperandBundles;
422 }
423 };
424
425
426 //===--------------------------------------------------------------------===//
427 // Miscellaneous creation methods.
428 //===--------------------------------------------------------------------===//
429
430 /// Make a new global variable with initializer type i8*
431 ///
432 /// Make a new global variable with an initializer that has array of i8 type
433 /// filled in with the null terminated string value specified. The new global
434 /// variable will be marked mergable with any others of the same contents. If
435 /// Name is specified, it is the name of the global variable created.
436 ///
437 /// If no module is given via \p M, it is take from the insertion point basic
438 /// block.
439 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
440 unsigned AddressSpace = 0,
441 Module *M = nullptr);
442
443 /// Get a constant value representing either true or false.
444 ConstantInt *getInt1(bool V) {
445 return ConstantInt::get(getInt1Ty(), V);
446 }
447
448 /// Get the constant value for i1 true.
449 ConstantInt *getTrue() {
450 return ConstantInt::getTrue(Context);
451 }
452
453 /// Get the constant value for i1 false.
454 ConstantInt *getFalse() {
455 return ConstantInt::getFalse(Context);
456 }
457
458 /// Get a constant 8-bit value.
459 ConstantInt *getInt8(uint8_t C) {
460 return ConstantInt::get(getInt8Ty(), C);
461 }
462
463 /// Get a constant 16-bit value.
464 ConstantInt *getInt16(uint16_t C) {
465 return ConstantInt::get(getInt16Ty(), C);
466 }
467
468 /// Get a constant 32-bit value.
469 ConstantInt *getInt32(uint32_t C) {
470 return ConstantInt::get(getInt32Ty(), C);
471 }
472
473 /// Get a constant 64-bit value.
474 ConstantInt *getInt64(uint64_t C) {
475 return ConstantInt::get(getInt64Ty(), C);
476 }
477
478 /// Get a constant N-bit value, zero extended or truncated from
479 /// a 64-bit value.
480 ConstantInt *getIntN(unsigned N, uint64_t C) {
481 return ConstantInt::get(getIntNTy(N), C);
482 }
483
484 /// Get a constant integer value.
485 ConstantInt *getInt(const APInt &AI) {
486 return ConstantInt::get(Context, AI);
487 }
488
489 //===--------------------------------------------------------------------===//
490 // Type creation methods
491 //===--------------------------------------------------------------------===//
492
493 /// Fetch the type representing a single bit
494 IntegerType *getInt1Ty() {
495 return Type::getInt1Ty(Context);
496 }
497
498 /// Fetch the type representing an 8-bit integer.
499 IntegerType *getInt8Ty() {
500 return Type::getInt8Ty(Context);
501 }
502
503 /// Fetch the type representing a 16-bit integer.
504 IntegerType *getInt16Ty() {
505 return Type::getInt16Ty(Context);
506 }
507
508 /// Fetch the type representing a 32-bit integer.
509 IntegerType *getInt32Ty() {
510 return Type::getInt32Ty(Context);
511 }
512
513 /// Fetch the type representing a 64-bit integer.
514 IntegerType *getInt64Ty() {
515 return Type::getInt64Ty(Context);
516 }
517
518 /// Fetch the type representing a 128-bit integer.
519 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
520
521 /// Fetch the type representing an N-bit integer.
522 IntegerType *getIntNTy(unsigned N) {
523 return Type::getIntNTy(Context, N);
524 }
525
526 /// Fetch the type representing a 16-bit floating point value.
527 Type *getHalfTy() {
528 return Type::getHalfTy(Context);
529 }
530
531 /// Fetch the type representing a 16-bit brain floating point value.
532 Type *getBFloatTy() {
533 return Type::getBFloatTy(Context);
534 }
535
536 /// Fetch the type representing a 32-bit floating point value.
537 Type *getFloatTy() {
538 return Type::getFloatTy(Context);
539 }
540
541 /// Fetch the type representing a 64-bit floating point value.
542 Type *getDoubleTy() {
543 return Type::getDoubleTy(Context);
544 }
545
546 /// Fetch the type representing void.
547 Type *getVoidTy() {
548 return Type::getVoidTy(Context);
549 }
550
551 /// Fetch the type representing a pointer.
552 PointerType *getPtrTy(unsigned AddrSpace = 0) {
553 return PointerType::get(Context, AddrSpace);
554 }
555
556 /// Fetch the type representing a pointer to an 8-bit integer value.
557 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
558 return Type::getInt8PtrTy(Context, AddrSpace);
559 }
560
561 /// Fetch the type representing a pointer to an integer value.
562 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
563 return DL.getIntPtrType(Context, AddrSpace);
564 }
565
566 //===--------------------------------------------------------------------===//
567 // Intrinsic creation methods
568 //===--------------------------------------------------------------------===//
569
570 /// Create and insert a memset to the specified pointer and the
571 /// specified value.
572 ///
573 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
574 /// specified, it will be added to the instruction. Likewise with alias.scope
575 /// and noalias tags.
576 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
577 MaybeAlign Align, bool isVolatile = false,
578 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
579 MDNode *NoAliasTag = nullptr) {
580 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
581 TBAATag, ScopeTag, NoAliasTag);
582 }
583
584 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
585 bool isVolatile = false, MDNode *TBAATag = nullptr,
586 MDNode *ScopeTag = nullptr,
587 MDNode *NoAliasTag = nullptr);
588
589 CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val,
590 Value *Size, bool IsVolatile = false,
591 MDNode *TBAATag = nullptr,
592 MDNode *ScopeTag = nullptr,
593 MDNode *NoAliasTag = nullptr);
594
595 /// Create and insert an element unordered-atomic memset of the region of
596 /// memory starting at the given pointer to the given value.
597 ///
598 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
599 /// specified, it will be added to the instruction. Likewise with alias.scope
600 /// and noalias tags.
601 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
602 uint64_t Size, Align Alignment,
603 uint32_t ElementSize,
604 MDNode *TBAATag = nullptr,
605 MDNode *ScopeTag = nullptr,
606 MDNode *NoAliasTag = nullptr) {
607 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
608 Align(Alignment), ElementSize,
609 TBAATag, ScopeTag, NoAliasTag);
610 }
611
612 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
613 Value *Size, Align Alignment,
614 uint32_t ElementSize,
615 MDNode *TBAATag = nullptr,
616 MDNode *ScopeTag = nullptr,
617 MDNode *NoAliasTag = nullptr);
618
619 /// Create and insert a memcpy between the specified pointers.
620 ///
621 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
622 /// specified, it will be added to the instruction. Likewise with alias.scope
623 /// and noalias tags.
624 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
625 MaybeAlign SrcAlign, uint64_t Size,
626 bool isVolatile = false, MDNode *TBAATag = nullptr,
627 MDNode *TBAAStructTag = nullptr,
628 MDNode *ScopeTag = nullptr,
629 MDNode *NoAliasTag = nullptr) {
630 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
631 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
632 NoAliasTag);
633 }
634
635 CallInst *CreateMemTransferInst(
636 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
637 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
638 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
639 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
640
641 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
642 MaybeAlign SrcAlign, Value *Size,
643 bool isVolatile = false, MDNode *TBAATag = nullptr,
644 MDNode *TBAAStructTag = nullptr,
645 MDNode *ScopeTag = nullptr,
646 MDNode *NoAliasTag = nullptr) {
647 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
648 SrcAlign, Size, isVolatile, TBAATag,
649 TBAAStructTag, ScopeTag, NoAliasTag);
650 }
651
652 CallInst *
653 CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
654 MaybeAlign SrcAlign, Value *Size, bool IsVolatile = false,
655 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
656 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
657
658 /// Create and insert an element unordered-atomic memcpy between the
659 /// specified pointers.
660 ///
661 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
662 ///
663 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
664 /// specified, it will be added to the instruction. Likewise with alias.scope
665 /// and noalias tags.
666 CallInst *CreateElementUnorderedAtomicMemCpy(
667 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
668 uint32_t ElementSize, MDNode *TBAATag = nullptr,
669 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
670 MDNode *NoAliasTag = nullptr);
671
672 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
673 MaybeAlign SrcAlign, uint64_t Size,
674 bool isVolatile = false, MDNode *TBAATag = nullptr,
675 MDNode *ScopeTag = nullptr,
676 MDNode *NoAliasTag = nullptr) {
677 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
678 isVolatile, TBAATag, ScopeTag, NoAliasTag);
679 }
680
681 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
682 MaybeAlign SrcAlign, Value *Size,
683 bool isVolatile = false, MDNode *TBAATag = nullptr,
684 MDNode *ScopeTag = nullptr,
685 MDNode *NoAliasTag = nullptr);
686
687 /// \brief Create and insert an element unordered-atomic memmove between the
688 /// specified pointers.
689 ///
690 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
691 /// respectively.
692 ///
693 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
694 /// specified, it will be added to the instruction. Likewise with alias.scope
695 /// and noalias tags.