Bug Summary

File:build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include/llvm/IR/IRBuilder.h
Warning:line 187, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name OMPIRBuilder.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Frontend/OpenMP -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Frontend/OpenMP -I include -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-09-04-125545-48738-1 -x c++ /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/CodeMetrics.h"
20#include "llvm/Analysis/LoopInfo.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ScalarEvolution.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/IR/CFG.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/GlobalVariable.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/PassManager.h"
31#include "llvm/IR/Value.h"
32#include "llvm/MC/TargetRegistry.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Target/TargetMachine.h"
35#include "llvm/Target/TargetOptions.h"
36#include "llvm/Transforms/Utils/BasicBlockUtils.h"
37#include "llvm/Transforms/Utils/Cloning.h"
38#include "llvm/Transforms/Utils/CodeExtractor.h"
39#include "llvm/Transforms/Utils/LoopPeel.h"
40#include "llvm/Transforms/Utils/UnrollLoop.h"
41
42#include <cstdint>
43
44#define DEBUG_TYPE"openmp-ir-builder" "openmp-ir-builder"
45
46using namespace llvm;
47using namespace omp;
48
49static cl::opt<bool>
50 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
51 cl::desc("Use optimistic attributes describing "
52 "'as-if' properties of runtime calls."),
53 cl::init(false));
54
55static cl::opt<double> UnrollThresholdFactor(
56 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
57 cl::desc("Factor for the unroll threshold to account for code "
58 "simplifications still taking place"),
59 cl::init(1.5));
60
61#ifndef NDEBUG
62/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
63/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
64/// an InsertPoint stores the instruction before something is inserted. For
65/// instance, if both point to the same instruction, two IRBuilders alternating
66/// creating instruction will cause the instructions to be interleaved.
67static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
68 IRBuilder<>::InsertPoint IP2) {
69 if (!IP1.isSet() || !IP2.isSet())
70 return false;
71 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
72}
73
74static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
75 // Valid ordered/unordered and base algorithm combinations.
76 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
77 case OMPScheduleType::UnorderedStaticChunked:
78 case OMPScheduleType::UnorderedStatic:
79 case OMPScheduleType::UnorderedDynamicChunked:
80 case OMPScheduleType::UnorderedGuidedChunked:
81 case OMPScheduleType::UnorderedRuntime:
82 case OMPScheduleType::UnorderedAuto:
83 case OMPScheduleType::UnorderedTrapezoidal:
84 case OMPScheduleType::UnorderedGreedy:
85 case OMPScheduleType::UnorderedBalanced:
86 case OMPScheduleType::UnorderedGuidedIterativeChunked:
87 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
88 case OMPScheduleType::UnorderedSteal:
89 case OMPScheduleType::UnorderedStaticBalancedChunked:
90 case OMPScheduleType::UnorderedGuidedSimd:
91 case OMPScheduleType::UnorderedRuntimeSimd:
92 case OMPScheduleType::OrderedStaticChunked:
93 case OMPScheduleType::OrderedStatic:
94 case OMPScheduleType::OrderedDynamicChunked:
95 case OMPScheduleType::OrderedGuidedChunked:
96 case OMPScheduleType::OrderedRuntime:
97 case OMPScheduleType::OrderedAuto:
98 case OMPScheduleType::OrderdTrapezoidal:
99 case OMPScheduleType::NomergeUnorderedStaticChunked:
100 case OMPScheduleType::NomergeUnorderedStatic:
101 case OMPScheduleType::NomergeUnorderedDynamicChunked:
102 case OMPScheduleType::NomergeUnorderedGuidedChunked:
103 case OMPScheduleType::NomergeUnorderedRuntime:
104 case OMPScheduleType::NomergeUnorderedAuto:
105 case OMPScheduleType::NomergeUnorderedTrapezoidal:
106 case OMPScheduleType::NomergeUnorderedGreedy:
107 case OMPScheduleType::NomergeUnorderedBalanced:
108 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
109 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
110 case OMPScheduleType::NomergeUnorderedSteal:
111 case OMPScheduleType::NomergeOrderedStaticChunked:
112 case OMPScheduleType::NomergeOrderedStatic:
113 case OMPScheduleType::NomergeOrderedDynamicChunked:
114 case OMPScheduleType::NomergeOrderedGuidedChunked:
115 case OMPScheduleType::NomergeOrderedRuntime:
116 case OMPScheduleType::NomergeOrderedAuto:
117 case OMPScheduleType::NomergeOrderedTrapezoidal:
118 break;
119 default:
120 return false;
121 }
122
123 // Must not set both monotonicity modifiers at the same time.
124 OMPScheduleType MonotonicityFlags =
125 SchedType & OMPScheduleType::MonotonicityMask;
126 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
127 return false;
128
129 return true;
130}
131#endif
132
133/// Determine which scheduling algorithm to use, determined from schedule clause
134/// arguments.
135static OMPScheduleType
136getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
137 bool HasSimdModifier) {
138 // Currently, the default schedule it static.
139 switch (ClauseKind) {
140 case OMP_SCHEDULE_Default:
141 case OMP_SCHEDULE_Static:
142 return HasChunks ? OMPScheduleType::BaseStaticChunked
143 : OMPScheduleType::BaseStatic;
144 case OMP_SCHEDULE_Dynamic:
145 return OMPScheduleType::BaseDynamicChunked;
146 case OMP_SCHEDULE_Guided:
147 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
148 : OMPScheduleType::BaseGuidedChunked;
149 case OMP_SCHEDULE_Auto:
150 return llvm::omp::OMPScheduleType::BaseAuto;
151 case OMP_SCHEDULE_Runtime:
152 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
153 : OMPScheduleType::BaseRuntime;
154 }
155 llvm_unreachable("unhandled schedule clause argument")::llvm::llvm_unreachable_internal("unhandled schedule clause argument"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 155)
;
156}
157
158/// Adds ordering modifier flags to schedule type.
159static OMPScheduleType
160getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
161 bool HasOrderedClause) {
162 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 164, __extension__
__PRETTY_FUNCTION__))
163 OMPScheduleType::None &&(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 164, __extension__
__PRETTY_FUNCTION__))
164 "Must not have ordering nor monotonicity flags already set")(static_cast <bool> ((BaseScheduleType & OMPScheduleType
::ModifierMask) == OMPScheduleType::None && "Must not have ordering nor monotonicity flags already set"
) ? void (0) : __assert_fail ("(BaseScheduleType & OMPScheduleType::ModifierMask) == OMPScheduleType::None && \"Must not have ordering nor monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 164, __extension__
__PRETTY_FUNCTION__))
;
165
166 OMPScheduleType OrderingModifier = HasOrderedClause
167 ? OMPScheduleType::ModifierOrdered
168 : OMPScheduleType::ModifierUnordered;
169 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
170
171 // Unsupported combinations
172 if (OrderingScheduleType ==
173 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
174 return OMPScheduleType::OrderedGuidedChunked;
175 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
176 OMPScheduleType::ModifierOrdered))
177 return OMPScheduleType::OrderedRuntime;
178
179 return OrderingScheduleType;
180}
181
182/// Adds monotonicity modifier flags to schedule type.
183static OMPScheduleType
184getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
185 bool HasSimdModifier, bool HasMonotonic,
186 bool HasNonmonotonic, bool HasOrderedClause) {
187 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 189, __extension__
__PRETTY_FUNCTION__))
188 OMPScheduleType::None &&(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 189, __extension__
__PRETTY_FUNCTION__))
189 "Must not have monotonicity flags already set")(static_cast <bool> ((ScheduleType & OMPScheduleType
::MonotonicityMask) == OMPScheduleType::None && "Must not have monotonicity flags already set"
) ? void (0) : __assert_fail ("(ScheduleType & OMPScheduleType::MonotonicityMask) == OMPScheduleType::None && \"Must not have monotonicity flags already set\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 189, __extension__
__PRETTY_FUNCTION__))
;
190 assert((!HasMonotonic || !HasNonmonotonic) &&(static_cast <bool> ((!HasMonotonic || !HasNonmonotonic
) && "Monotonic and Nonmonotonic are contradicting each other"
) ? void (0) : __assert_fail ("(!HasMonotonic || !HasNonmonotonic) && \"Monotonic and Nonmonotonic are contradicting each other\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 191, __extension__
__PRETTY_FUNCTION__))
191 "Monotonic and Nonmonotonic are contradicting each other")(static_cast <bool> ((!HasMonotonic || !HasNonmonotonic
) && "Monotonic and Nonmonotonic are contradicting each other"
) ? void (0) : __assert_fail ("(!HasMonotonic || !HasNonmonotonic) && \"Monotonic and Nonmonotonic are contradicting each other\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 191, __extension__
__PRETTY_FUNCTION__))
;
192
193 if (HasMonotonic) {
194 return ScheduleType | OMPScheduleType::ModifierMonotonic;
195 } else if (HasNonmonotonic) {
196 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
197 } else {
198 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
199 // If the static schedule kind is specified or if the ordered clause is
200 // specified, and if the nonmonotonic modifier is not specified, the
201 // effect is as if the monotonic modifier is specified. Otherwise, unless
202 // the monotonic modifier is specified, the effect is as if the
203 // nonmonotonic modifier is specified.
204 OMPScheduleType BaseScheduleType =
205 ScheduleType & ~OMPScheduleType::ModifierMask;
206 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
207 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
208 HasOrderedClause) {
209 // The monotonic is used by default in openmp runtime library, so no need
210 // to set it.
211 return ScheduleType;
212 } else {
213 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
214 }
215 }
216}
217
218/// Determine the schedule type using schedule and ordering clause arguments.
219static OMPScheduleType
220computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
221 bool HasSimdModifier, bool HasMonotonicModifier,
222 bool HasNonmonotonicModifier, bool HasOrderedClause) {
223 OMPScheduleType BaseSchedule =
224 getOpenMPBaseScheduleType(ClauseKind, HasChunks, HasSimdModifier);
225 OMPScheduleType OrderedSchedule =
226 getOpenMPOrderingScheduleType(BaseSchedule, HasOrderedClause);
227 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
228 OrderedSchedule, HasSimdModifier, HasMonotonicModifier,
229 HasNonmonotonicModifier, HasOrderedClause);
230
231 assert(isValidWorkshareLoopScheduleType(Result))(static_cast <bool> (isValidWorkshareLoopScheduleType(Result
)) ? void (0) : __assert_fail ("isValidWorkshareLoopScheduleType(Result)"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 231, __extension__
__PRETTY_FUNCTION__))
;
232 return Result;
233}
234
235/// Make \p Source branch to \p Target.
236///
237/// Handles two situations:
238/// * \p Source already has an unconditional branch.
239/// * \p Source is a degenerate block (no terminator because the BB is
240/// the current head of the IR construction).
241static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
242 if (Instruction *Term = Source->getTerminator()) {
243 auto *Br = cast<BranchInst>(Term);
244 assert(!Br->isConditional() &&(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 245, __extension__
__PRETTY_FUNCTION__))
245 "BB's terminator must be an unconditional branch (or degenerate)")(static_cast <bool> (!Br->isConditional() &&
"BB's terminator must be an unconditional branch (or degenerate)"
) ? void (0) : __assert_fail ("!Br->isConditional() && \"BB's terminator must be an unconditional branch (or degenerate)\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 245, __extension__
__PRETTY_FUNCTION__))
;
246 BasicBlock *Succ = Br->getSuccessor(0);
247 Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
248 Br->setSuccessor(0, Target);
249 return;
250 }
251
252 auto *NewBr = BranchInst::Create(Target, Source);
253 NewBr->setDebugLoc(DL);
254}
255
256void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
257 bool CreateBranch) {
258 assert(New->getFirstInsertionPt() == New->begin() &&(static_cast <bool> (New->getFirstInsertionPt() == New
->begin() && "Target BB must not have PHI nodes") ?
void (0) : __assert_fail ("New->getFirstInsertionPt() == New->begin() && \"Target BB must not have PHI nodes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 259, __extension__
__PRETTY_FUNCTION__))
259 "Target BB must not have PHI nodes")(static_cast <bool> (New->getFirstInsertionPt() == New
->begin() && "Target BB must not have PHI nodes") ?
void (0) : __assert_fail ("New->getFirstInsertionPt() == New->begin() && \"Target BB must not have PHI nodes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 259, __extension__
__PRETTY_FUNCTION__))
;
260
261 // Move instructions to new block.
262 BasicBlock *Old = IP.getBlock();
263 New->getInstList().splice(New->begin(), Old->getInstList(), IP.getPoint(),
264 Old->end());
265
266 if (CreateBranch)
267 BranchInst::Create(New, Old);
268}
269
270void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
271 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
272 BasicBlock *Old = Builder.GetInsertBlock();
273
274 spliceBB(Builder.saveIP(), New, CreateBranch);
275 if (CreateBranch)
276 Builder.SetInsertPoint(Old->getTerminator());
277 else
278 Builder.SetInsertPoint(Old);
279
280 // SetInsertPoint also updates the Builder's debug location, but we want to
281 // keep the one the Builder was configured to use.
282 Builder.SetCurrentDebugLocation(DebugLoc);
283}
284
285BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
286 llvm::Twine Name) {
287 BasicBlock *Old = IP.getBlock();
288 BasicBlock *New = BasicBlock::Create(
289 Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name,
290 Old->getParent(), Old->getNextNode());
291 spliceBB(IP, New, CreateBranch);
292 New->replaceSuccessorsPhiUsesWith(Old, New);
293 return New;
294}
295
296BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
297 llvm::Twine Name) {
298 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
299 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
300 if (CreateBranch)
301 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
302 else
303 Builder.SetInsertPoint(Builder.GetInsertBlock());
304 // SetInsertPoint also updates the Builder's debug location, but we want to
305 // keep the one the Builder was configured to use.
306 Builder.SetCurrentDebugLocation(DebugLoc);
307 return New;
308}
309
310BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
311 llvm::Twine Name) {
312 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
313 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
314 if (CreateBranch)
315 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
316 else
317 Builder.SetInsertPoint(Builder.GetInsertBlock());
318 // SetInsertPoint also updates the Builder's debug location, but we want to
319 // keep the one the Builder was configured to use.
320 Builder.SetCurrentDebugLocation(DebugLoc);
321 return New;
322}
323
324BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
325 llvm::Twine Suffix) {
326 BasicBlock *Old = Builder.GetInsertBlock();
327 return splitBB(Builder, CreateBranch, Old->getName() + Suffix);
328}
329
330void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
331 LLVMContext &Ctx = Fn.getContext();
332
333 // Get the function's current attributes.
334 auto Attrs = Fn.getAttributes();
335 auto FnAttrs = Attrs.getFnAttrs();
336 auto RetAttrs = Attrs.getRetAttrs();
337 SmallVector<AttributeSet, 4> ArgAttrs;
338 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
339 ArgAttrs.emplace_back(Attrs.getParamAttrs(ArgNo));
340
341#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
342#include "llvm/Frontend/OpenMP/OMPKinds.def"
343
344 // Add attributes to the function declaration.
345 switch (FnID) {
346#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
347 case Enum: \
348 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
349 RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
350 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
351 ArgAttrs[ArgNo] = \
352 ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
353 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
354 break;
355#include "llvm/Frontend/OpenMP/OMPKinds.def"
356 default:
357 // Attributes are optional.
358 break;
359 }
360}
361
362FunctionCallee
363OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
364 FunctionType *FnTy = nullptr;
365 Function *Fn = nullptr;
366
367 // Try to find the declation in the module first.
368 switch (FnID) {
369#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
370 case Enum: \
371 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
372 IsVarArg); \
373 Fn = M.getFunction(Str); \
374 break;
375#include "llvm/Frontend/OpenMP/OMPKinds.def"
376 }
377
378 if (!Fn) {
379 // Create a new declaration if we need one.
380 switch (FnID) {
381#define OMP_RTL(Enum, Str, ...) \
382 case Enum: \
383 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
384 break;
385#include "llvm/Frontend/OpenMP/OMPKinds.def"
386 }
387
388 // Add information if the runtime function takes a callback function
389 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
390 if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
391 LLVMContext &Ctx = Fn->getContext();
392 MDBuilder MDB(Ctx);
393 // Annotate the callback behavior of the runtime function:
394 // - The callback callee is argument number 2 (microtask).
395 // - The first two arguments of the callback callee are unknown (-1).
396 // - All variadic arguments to the runtime function are passed to the
397 // callback callee.
398 Fn->addMetadata(
399 LLVMContext::MD_callback,
400 *MDNode::get(Ctx, {MDB.createCallbackEncoding(
401 2, {-1, -1}, /* VarArgsArePassed */ true)}));
402 }
403 }
404
405 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
406 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Created OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
407 addAttributes(FnID, *Fn);
408
409 } else {
410 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
411 << " with type " << *Fn->getFunctionType() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Found OpenMP runtime function "
<< Fn->getName() << " with type " << *Fn
->getFunctionType() << "\n"; } } while (false)
;
412 }
413
414 assert(Fn && "Failed to create OpenMP runtime function")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 414, __extension__
__PRETTY_FUNCTION__))
;
415
416 // Cast the function to the expected type if necessary
417 Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo());
418 return {FnTy, C};
419}
420
421Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
422 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
423 auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
424 assert(Fn && "Failed to create OpenMP runtime function pointer")(static_cast <bool> (Fn && "Failed to create OpenMP runtime function pointer"
) ? void (0) : __assert_fail ("Fn && \"Failed to create OpenMP runtime function pointer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 424, __extension__
__PRETTY_FUNCTION__))
;
425 return Fn;
426}
427
428void OpenMPIRBuilder::initialize() { initializeTypes(M); }
429
430void OpenMPIRBuilder::finalize(Function *Fn) {
431 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
432 SmallVector<BasicBlock *, 32> Blocks;
433 SmallVector<OutlineInfo, 16> DeferredOutlines;
434 for (OutlineInfo &OI : OutlineInfos) {
435 // Skip functions that have not finalized yet; may happen with nested
436 // function generation.
437 if (Fn && OI.getFunction() != Fn) {
438 DeferredOutlines.push_back(OI);
439 continue;
440 }
441
442 ParallelRegionBlockSet.clear();
443 Blocks.clear();
444 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
445
446 Function *OuterFn = OI.getFunction();
447 CodeExtractorAnalysisCache CEAC(*OuterFn);
448 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
449 /* AggregateArgs */ true,
450 /* BlockFrequencyInfo */ nullptr,
451 /* BranchProbabilityInfo */ nullptr,
452 /* AssumptionCache */ nullptr,
453 /* AllowVarArgs */ true,
454 /* AllowAlloca */ true,
455 /* AllocaBlock*/ OI.OuterAllocaBB,
456 /* Suffix */ ".omp_par");
457
458 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before outlining: "
<< *OuterFn << "\n"; } } while (false)
;
459 LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
460 << " Exit: " << OI.ExitBB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Entry " << OI
.EntryBB->getName() << " Exit: " << OI.ExitBB->
getName() << "\n"; } } while (false)
;
461 assert(Extractor.isEligible() &&(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 462, __extension__
__PRETTY_FUNCTION__))
462 "Expected OpenMP outlining to be possible!")(static_cast <bool> (Extractor.isEligible() && "Expected OpenMP outlining to be possible!"
) ? void (0) : __assert_fail ("Extractor.isEligible() && \"Expected OpenMP outlining to be possible!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 462, __extension__
__PRETTY_FUNCTION__))
;
463
464 for (auto *V : OI.ExcludeArgsFromAggregate)
465 Extractor.excludeArgFromAggregate(V);
466
467 Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
468
469 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After outlining: "
<< *OuterFn << "\n"; } } while (false)
;
470 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << " Outlined function: "
<< *OutlinedFn << "\n"; } } while (false)
;
471 assert(OutlinedFn->getReturnType()->isVoidTy() &&(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 472, __extension__
__PRETTY_FUNCTION__))
472 "OpenMP outlined functions should not return a value!")(static_cast <bool> (OutlinedFn->getReturnType()->
isVoidTy() && "OpenMP outlined functions should not return a value!"
) ? void (0) : __assert_fail ("OutlinedFn->getReturnType()->isVoidTy() && \"OpenMP outlined functions should not return a value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 472, __extension__
__PRETTY_FUNCTION__))
;
473
474 // For compability with the clang CG we move the outlined function after the
475 // one with the parallel region.
476 OutlinedFn->removeFromParent();
477 M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);
478
479 // Remove the artificial entry introduced by the extractor right away, we
480 // made our own entry block after all.
481 {
482 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
483 assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB)(static_cast <bool> (ArtificialEntry.getUniqueSuccessor
() == OI.EntryBB) ? void (0) : __assert_fail ("ArtificialEntry.getUniqueSuccessor() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 483, __extension__
__PRETTY_FUNCTION__))
;
484 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry)(static_cast <bool> (OI.EntryBB->getUniquePredecessor
() == &ArtificialEntry) ? void (0) : __assert_fail ("OI.EntryBB->getUniquePredecessor() == &ArtificialEntry"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 484, __extension__
__PRETTY_FUNCTION__))
;
485 // Move instructions from the to-be-deleted ArtificialEntry to the entry
486 // basic block of the parallel region. CodeExtractor generates
487 // instructions to unwrap the aggregate argument and may sink
488 // allocas/bitcasts for values that are solely used in the outlined region
489 // and do not escape.
490 assert(!ArtificialEntry.empty() &&(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to add in the outlined region entry")
? void (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to add in the outlined region entry\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 491, __extension__
__PRETTY_FUNCTION__))
491 "Expected instructions to add in the outlined region entry")(static_cast <bool> (!ArtificialEntry.empty() &&
"Expected instructions to add in the outlined region entry")
? void (0) : __assert_fail ("!ArtificialEntry.empty() && \"Expected instructions to add in the outlined region entry\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 491, __extension__
__PRETTY_FUNCTION__))
;
492 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
493 End = ArtificialEntry.rend();
494 It != End;) {
495 Instruction &I = *It;
496 It++;
497
498 if (I.isTerminator())
499 continue;
500
501 I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
502 }
503
504 OI.EntryBB->moveBefore(&ArtificialEntry);
505 ArtificialEntry.eraseFromParent();
506 }
507 assert(&OutlinedFn->getEntryBlock() == OI.EntryBB)(static_cast <bool> (&OutlinedFn->getEntryBlock(
) == OI.EntryBB) ? void (0) : __assert_fail ("&OutlinedFn->getEntryBlock() == OI.EntryBB"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 507, __extension__
__PRETTY_FUNCTION__))
;
508 assert(OutlinedFn && OutlinedFn->getNumUses() == 1)(static_cast <bool> (OutlinedFn && OutlinedFn->
getNumUses() == 1) ? void (0) : __assert_fail ("OutlinedFn && OutlinedFn->getNumUses() == 1"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 508, __extension__
__PRETTY_FUNCTION__))
;
509
510 // Run a user callback, e.g. to add attributes.
511 if (OI.PostOutlineCB)
512 OI.PostOutlineCB(*OutlinedFn);
513 }
514
515 // Remove work items that have been completed.
516 OutlineInfos = std::move(DeferredOutlines);
517}
518
519OpenMPIRBuilder::~OpenMPIRBuilder() {
520 assert(OutlineInfos.empty() && "There must be no outstanding outlinings")(static_cast <bool> (OutlineInfos.empty() && "There must be no outstanding outlinings"
) ? void (0) : __assert_fail ("OutlineInfos.empty() && \"There must be no outstanding outlinings\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 520, __extension__
__PRETTY_FUNCTION__))
;
521}
522
523GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
524 IntegerType *I32Ty = Type::getInt32Ty(M.getContext());
525 auto *GV =
526 new GlobalVariable(M, I32Ty,
527 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
528 ConstantInt::get(I32Ty, Value), Name);
529 GV->setVisibility(GlobalValue::HiddenVisibility);
530
531 return GV;
532}
533
534Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
535 uint32_t SrcLocStrSize,
536 IdentFlag LocFlags,
537 unsigned Reserve2Flags) {
538 // Enable "C-mode".
539 LocFlags |= OMP_IDENT_FLAG_KMPC;
540
541 Constant *&Ident =
542 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
543 if (!Ident) {
544 Constant *I32Null = ConstantInt::getNullValue(Int32);
545 Constant *IdentData[] = {I32Null,
546 ConstantInt::get(Int32, uint32_t(LocFlags)),
547 ConstantInt::get(Int32, Reserve2Flags),
548 ConstantInt::get(Int32, SrcLocStrSize), SrcLocStr};
549 Constant *Initializer =
550 ConstantStruct::get(OpenMPIRBuilder::Ident, IdentData);
551
552 // Look for existing encoding of the location + flags, not needed but
553 // minimizes the difference to the existing solution while we transition.
554 for (GlobalVariable &GV : M.getGlobalList())
555 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
556 if (GV.getInitializer() == Initializer)
557 Ident = &GV;
558
559 if (!Ident) {
560 auto *GV = new GlobalVariable(
561 M, OpenMPIRBuilder::Ident,
562 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
563 nullptr, GlobalValue::NotThreadLocal,
564 M.getDataLayout().getDefaultGlobalsAddressSpace());
565 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
566 GV->setAlignment(Align(8));
567 Ident = GV;
568 }
569 }
570
571 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(Ident, IdentPtr);
572}
573
574Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
575 uint32_t &SrcLocStrSize) {
576 SrcLocStrSize = LocStr.size();
577 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
578 if (!SrcLocStr) {
579 Constant *Initializer =
580 ConstantDataArray::getString(M.getContext(), LocStr);
581
582 // Look for existing encoding of the location, not needed but minimizes the
583 // difference to the existing solution while we transition.
584 for (GlobalVariable &GV : M.getGlobalList())
585 if (GV.isConstant() && GV.hasInitializer() &&
586 GV.getInitializer() == Initializer)
587 return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);
588
589 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
590 /* AddressSpace */ 0, &M);
591 }
592 return SrcLocStr;
593}
594
595Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
596 StringRef FileName,
597 unsigned Line, unsigned Column,
598 uint32_t &SrcLocStrSize) {
599 SmallString<128> Buffer;
600 Buffer.push_back(';');
601 Buffer.append(FileName);
602 Buffer.push_back(';');
603 Buffer.append(FunctionName);
604 Buffer.push_back(';');
605 Buffer.append(std::to_string(Line));
606 Buffer.push_back(';');
607 Buffer.append(std::to_string(Column));
608 Buffer.push_back(';');
609 Buffer.push_back(';');
610 return getOrCreateSrcLocStr(Buffer.str(), SrcLocStrSize);
611}
612
613Constant *
614OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
615 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
616 return getOrCreateSrcLocStr(UnknownLoc, SrcLocStrSize);
617}
618
619Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
620 uint32_t &SrcLocStrSize,
621 Function *F) {
622 DILocation *DIL = DL.get();
623 if (!DIL)
624 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
625 StringRef FileName = M.getName();
626 if (DIFile *DIF = DIL->getFile())
627 if (Optional<StringRef> Source = DIF->getSource())
628 FileName = *Source;
629 StringRef Function = DIL->getScope()->getSubprogram()->getName();
630 if (Function.empty() && F)
631 Function = F->getName();
632 return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
633 DIL->getColumn(), SrcLocStrSize);
634}
635
636Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
637 uint32_t &SrcLocStrSize) {
638 return getOrCreateSrcLocStr(Loc.DL, SrcLocStrSize,
639 Loc.IP.getBlock()->getParent());
640}
641
642Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
643 return Builder.CreateCall(
644 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
645 "omp_global_thread_num");
646}
647
648OpenMPIRBuilder::InsertPointTy
649OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
650 bool ForceSimpleCall, bool CheckCancelFlag) {
651 if (!updateToLocation(Loc))
652 return Loc.IP;
653 return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
654}
655
656OpenMPIRBuilder::InsertPointTy
657OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
658 bool ForceSimpleCall, bool CheckCancelFlag) {
659 // Build call __kmpc_cancel_barrier(loc, thread_id) or
660 // __kmpc_barrier(loc, thread_id);
661
662 IdentFlag BarrierLocFlags;
663 switch (Kind) {
664 case OMPD_for:
665 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
666 break;
667 case OMPD_sections:
668 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
669 break;
670 case OMPD_single:
671 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
672 break;
673 case OMPD_barrier:
674 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
675 break;
676 default:
677 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
678 break;
679 }
680
681 uint32_t SrcLocStrSize;
682 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
683 Value *Args[] = {
684 getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
685 getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
686
687 // If we are in a cancellable parallel region, barriers are cancellation
688 // points.
689 // TODO: Check why we would force simple calls or to ignore the cancel flag.
690 bool UseCancelBarrier =
691 !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
692
693 Value *Result =
694 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
695 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
696 : OMPRTL___kmpc_barrier),
697 Args);
698
699 if (UseCancelBarrier && CheckCancelFlag)
700 emitCancelationCheckImpl(Result, OMPD_parallel);
701
702 return Builder.saveIP();
703}
704
705OpenMPIRBuilder::InsertPointTy
706OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
707 Value *IfCondition,
708 omp::Directive CanceledDirective) {
709 if (!updateToLocation(Loc))
710 return Loc.IP;
711
712 // LLVM utilities like blocks with terminators.
713 auto *UI = Builder.CreateUnreachable();
714
715 Instruction *ThenTI = UI, *ElseTI = nullptr;
716 if (IfCondition)
717 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
718 Builder.SetInsertPoint(ThenTI);
719
720 Value *CancelKind = nullptr;
721 switch (CanceledDirective) {
722#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
723 case DirectiveEnum: \
724 CancelKind = Builder.getInt32(Value); \
725 break;
726#include "llvm/Frontend/OpenMP/OMPKinds.def"
727 default:
728 llvm_unreachable("Unknown cancel kind!")::llvm::llvm_unreachable_internal("Unknown cancel kind!", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 728)
;
729 }
730
731 uint32_t SrcLocStrSize;
732 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
733 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
734 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
735 Value *Result = Builder.CreateCall(
736 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
737 auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
738 if (CanceledDirective == OMPD_parallel) {
739 IRBuilder<>::InsertPointGuard IPG(Builder);
740 Builder.restoreIP(IP);
741 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
742 omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
743 /* CheckCancelFlag */ false);
744 }
745 };
746
747 // The actual cancel logic is shared with others, e.g., cancel_barriers.
748 emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);
749
750 // Update the insertion point and remove the terminator we introduced.
751 Builder.SetInsertPoint(UI->getParent());
752 UI->eraseFromParent();
753
754 return Builder.saveIP();
755}
756
757void OpenMPIRBuilder::emitOffloadingEntry(Constant *Addr, StringRef Name,
758 uint64_t Size, int32_t Flags,
759 StringRef SectionName) {
760 Type *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
761 Type *Int32Ty = Type::getInt32Ty(M.getContext());
762 Type *SizeTy = M.getDataLayout().getIntPtrType(M.getContext());
763
764 Constant *AddrName = ConstantDataArray::getString(M.getContext(), Name);
765
766 // Create the constant string used to look up the symbol in the device.
767 auto *Str =
768 new llvm::GlobalVariable(M, AddrName->getType(), /*isConstant=*/true,
769 llvm::GlobalValue::InternalLinkage, AddrName,
770 ".omp_offloading.entry_name");
771 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
772
773 // Construct the offloading entry.
774 Constant *EntryData[] = {
775 ConstantExpr::getPointerBitCastOrAddrSpaceCast(Addr, Int8PtrTy),
776 ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, Int8PtrTy),
777 ConstantInt::get(SizeTy, Size),
778 ConstantInt::get(Int32Ty, Flags),
779 ConstantInt::get(Int32Ty, 0),
780 };
781 Constant *EntryInitializer =
782 ConstantStruct::get(OpenMPIRBuilder::OffloadEntry, EntryData);
783
784 auto *Entry = new GlobalVariable(
785 M, OpenMPIRBuilder::OffloadEntry,
786 /* isConstant = */ true, GlobalValue::WeakAnyLinkage, EntryInitializer,
787 ".omp_offloading.entry." + Name, nullptr, GlobalValue::NotThreadLocal,
788 M.getDataLayout().getDefaultGlobalsAddressSpace());
789
790 // The entry has to be created in the section the linker expects it to be.
791 Entry->setSection(SectionName);
792 Entry->setAlignment(Align(1));
793}
794
795OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
796 const LocationDescription &Loc, Value *&Return, Value *Ident,
797 Value *DeviceID, Value *NumTeams, Value *NumThreads, Value *HostPtr,
798 ArrayRef<Value *> KernelArgs, ArrayRef<Value *> NoWaitArgs) {
799 if (!updateToLocation(Loc))
800 return Loc.IP;
801
802 auto *KernelArgsPtr =
803 Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");
804 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
805 llvm::Value *Arg =
806 Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I);
807 Builder.CreateAlignedStore(
808 KernelArgs[I], Arg,
809 M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType()));
810 }
811
812 bool HasNoWait = !NoWaitArgs.empty();
813 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
814 NumThreads, HostPtr, KernelArgsPtr};
815 if (HasNoWait)
816 OffloadingArgs.append(NoWaitArgs.begin(), NoWaitArgs.end());
817
818 Return = Builder.CreateCall(
819 HasNoWait
820 ? getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel_nowait)
821 : getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel),
822 OffloadingArgs);
823
824 return Builder.saveIP();
825}
826
827void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
828 omp::Directive CanceledDirective,
829 FinalizeCallbackTy ExitCB) {
830 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 831, __extension__
__PRETTY_FUNCTION__))
831 "Unexpected cancellation!")(static_cast <bool> (isLastFinalizationInfoCancellable(
CanceledDirective) && "Unexpected cancellation!") ? void
(0) : __assert_fail ("isLastFinalizationInfoCancellable(CanceledDirective) && \"Unexpected cancellation!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 831, __extension__
__PRETTY_FUNCTION__))
;
832
833 // For a cancel barrier we create two new blocks.
834 BasicBlock *BB = Builder.GetInsertBlock();
835 BasicBlock *NonCancellationBlock;
836 if (Builder.GetInsertPoint() == BB->end()) {
837 // TODO: This branch will not be needed once we moved to the
838 // OpenMPIRBuilder codegen completely.
839 NonCancellationBlock = BasicBlock::Create(
840 BB->getContext(), BB->getName() + ".cont", BB->getParent());
841 } else {
842 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
843 BB->getTerminator()->eraseFromParent();
844 Builder.SetInsertPoint(BB);
845 }
846 BasicBlock *CancellationBlock = BasicBlock::Create(
847 BB->getContext(), BB->getName() + ".cncl", BB->getParent());
848
849 // Jump to them based on the return value.
850 Value *Cmp = Builder.CreateIsNull(CancelFlag);
851 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
852 /* TODO weight */ nullptr, nullptr);
853
854 // From the cancellation block we finalize all variables and go to the
855 // post finalization block that is known to the FiniCB callback.
856 Builder.SetInsertPoint(CancellationBlock);
857 if (ExitCB)
858 ExitCB(Builder.saveIP());
859 auto &FI = FinalizationStack.back();
860 FI.FiniCB(Builder.saveIP());
861
862 // The continuation block is where code generation continues.
863 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
864}
865
866IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
867 const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
868 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
869 FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
870 omp::ProcBindKind ProcBind, bool IsCancellable) {
871 assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous")(static_cast <bool> (!isConflictIP(Loc.IP, OuterAllocaIP
) && "IPs must not be ambiguous") ? void (0) : __assert_fail
("!isConflictIP(Loc.IP, OuterAllocaIP) && \"IPs must not be ambiguous\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 871, __extension__
__PRETTY_FUNCTION__))
;
872
873 if (!updateToLocation(Loc))
874 return Loc.IP;
875
876 uint32_t SrcLocStrSize;
877 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
878 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
879 Value *ThreadID = getOrCreateThreadID(Ident);
880
881 if (NumThreads) {
882 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
883 Value *Args[] = {
884 Ident, ThreadID,
885 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
886 Builder.CreateCall(
887 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
888 }
889
890 if (ProcBind != OMP_PROC_BIND_default) {
891 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
892 Value *Args[] = {
893 Ident, ThreadID,
894 ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
895 Builder.CreateCall(
896 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
897 }
898
899 BasicBlock *InsertBB = Builder.GetInsertBlock();
900 Function *OuterFn = InsertBB->getParent();
901
902 // Save the outer alloca block because the insertion iterator may get
903 // invalidated and we still need this later.
904 BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
905
906 // Vector to remember instructions we used only during the modeling but which
907 // we want to delete at the end.
908 SmallVector<Instruction *, 4> ToBeDeleted;
909
910 // Change the location to the outer alloca insertion point to create and
911 // initialize the allocas we pass into the parallel region.
912 Builder.restoreIP(OuterAllocaIP);
913 AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
914 AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
915
916 // If there is an if condition we actually use the TIDAddr and ZeroAddr in the
917 // program, otherwise we only need them for modeling purposes to get the
918 // associated arguments in the outlined function. In the former case,
919 // initialize the allocas properly, in the latter case, delete them later.
920 if (IfCondition) {
921 Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
922 Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
923 } else {
924 ToBeDeleted.push_back(TIDAddr);
925 ToBeDeleted.push_back(ZeroAddr);
926 }
927
928 // Create an artificial insertion point that will also ensure the blocks we
929 // are about to split are not degenerated.
930 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
931
932 Instruction *ThenTI = UI, *ElseTI = nullptr;
933 if (IfCondition)
934 SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
935
936 BasicBlock *ThenBB = ThenTI->getParent();
937 BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
938 BasicBlock *PRegBodyBB =
939 PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
940 BasicBlock *PRegPreFiniBB =
941 PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
942 BasicBlock *PRegExitBB =
943 PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");
944
945 auto FiniCBWrapper = [&](InsertPointTy IP) {
946 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
947 // target to the region exit block.
948 if (IP.getBlock()->end() == IP.getPoint()) {
949 IRBuilder<>::InsertPointGuard IPG(Builder);
950 Builder.restoreIP(IP);
951 Instruction *I = Builder.CreateBr(PRegExitBB);
952 IP = InsertPointTy(I->getParent(), I->getIterator());
953 }
954 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 956, __extension__
__PRETTY_FUNCTION__))
955 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 956, __extension__
__PRETTY_FUNCTION__))
956 "Unexpected insertion point for finalization call!")(static_cast <bool> (IP.getBlock()->getTerminator()->
getNumSuccessors() == 1 && IP.getBlock()->getTerminator
()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"
) ? void (0) : __assert_fail ("IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && \"Unexpected insertion point for finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 956, __extension__
__PRETTY_FUNCTION__))
;
957 return FiniCB(IP);
958 };
959
960 FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
961
962 // Generate the privatization allocas in the block that will become the entry
963 // of the outlined function.
964 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
965 InsertPointTy InnerAllocaIP = Builder.saveIP();
966
967 AllocaInst *PrivTIDAddr =
968 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
969 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
970
971 // Add some fake uses for OpenMP provided arguments.
972 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
973 Instruction *ZeroAddrUse =
974 Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");
975 ToBeDeleted.push_back(ZeroAddrUse);
976
977 // ThenBB
978 // |
979 // V
980 // PRegionEntryBB <- Privatization allocas are placed here.
981 // |
982 // V
983 // PRegionBodyBB <- BodeGen is invoked here.
984 // |
985 // V
986 // PRegPreFiniBB <- The block we will start finalization from.
987 // |
988 // V
989 // PRegionExitBB <- A common exit to simplify block collection.
990 //
991
992 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
993
994 // Let the caller create the body.
995 assert(BodyGenCB && "Expected body generation callback!")(static_cast <bool> (BodyGenCB && "Expected body generation callback!"
) ? void (0) : __assert_fail ("BodyGenCB && \"Expected body generation callback!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 995, __extension__
__PRETTY_FUNCTION__))
;
996 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
997 BodyGenCB(InnerAllocaIP, CodeGenIP);
998
999 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After body codegen: "
<< *OuterFn << "\n"; } } while (false)
;
1000
1001 FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
1002 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1003 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1004 llvm::LLVMContext &Ctx = F->getContext();
1005 MDBuilder MDB(Ctx);
1006 // Annotate the callback behavior of the __kmpc_fork_call:
1007 // - The callback callee is argument number 2 (microtask).
1008 // - The first two arguments of the callback callee are unknown (-1).
1009 // - All variadic arguments to the __kmpc_fork_call are passed to the
1010 // callback callee.
1011 F->addMetadata(
1012 llvm::LLVMContext::MD_callback,
1013 *llvm::MDNode::get(
1014 Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
1015 /* VarArgsArePassed */ true)}));
1016 }
1017 }
1018
1019 OutlineInfo OI;
1020 OI.PostOutlineCB = [=](Function &OutlinedFn) {
1021 // Add some known attributes.
1022 OutlinedFn.addParamAttr(0, Attribute::NoAlias);
1023 OutlinedFn.addParamAttr(1, Attribute::NoAlias);
1024 OutlinedFn.addFnAttr(Attribute::NoUnwind);
1025 OutlinedFn.addFnAttr(Attribute::NoRecurse);
1026
1027 assert(OutlinedFn.arg_size() >= 2 &&(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1028, __extension__
__PRETTY_FUNCTION__))
1028 "Expected at least tid and bounded tid as arguments")(static_cast <bool> (OutlinedFn.arg_size() >= 2 &&
"Expected at least tid and bounded tid as arguments") ? void
(0) : __assert_fail ("OutlinedFn.arg_size() >= 2 && \"Expected at least tid and bounded tid as arguments\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1028, __extension__
__PRETTY_FUNCTION__))
;
1029 unsigned NumCapturedVars =
1030 OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1031
1032 CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
1033 CI->getParent()->setName("omp_parallel");
1034 Builder.SetInsertPoint(CI);
1035
1036 // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
1037 Value *ForkCallArgs[] = {
1038 Ident, Builder.getInt32(NumCapturedVars),
1039 Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
1040
1041 SmallVector<Value *, 16> RealArgs;
1042 RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
1043 RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());
1044
1045 Builder.CreateCall(RTLFn, RealArgs);
1046
1047 LLVM_DEBUG(dbgs() << "With fork_call placed: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
1048 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With fork_call placed: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
1049
1050 InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());
1051
1052 // Initialize the local TID stack location with the argument value.
1053 Builder.SetInsertPoint(PrivTID);
1054 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1055 Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
1056
1057 // If no "if" clause was present we do not need the call created during
1058 // outlining, otherwise we reuse it in the serialized parallel region.
1059 if (!ElseTI) {
1060 CI->eraseFromParent();
1061 } else {
1062
1063 // If an "if" clause was present we are now generating the serialized
1064 // version into the "else" branch.
1065 Builder.SetInsertPoint(ElseTI);
1066
1067 // Build calls __kmpc_serialized_parallel(&Ident, GTid);
1068 Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
1069 Builder.CreateCall(
1070 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
1071 SerializedParallelCallArgs);
1072
1073 // OutlinedFn(&GTid, &zero, CapturedStruct);
1074 CI->removeFromParent();
1075 Builder.Insert(CI);
1076
1077 // __kmpc_end_serialized_parallel(&Ident, GTid);
1078 Value *EndArgs[] = {Ident, ThreadID};
1079 Builder.CreateCall(
1080 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
1081 EndArgs);
1082
1083 LLVM_DEBUG(dbgs() << "With serialized parallel region: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
1084 << *Builder.GetInsertBlock()->getParent() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "With serialized parallel region: "
<< *Builder.GetInsertBlock()->getParent() << "\n"
; } } while (false)
;
1085 }
1086
1087 for (Instruction *I : ToBeDeleted)
1088 I->eraseFromParent();
1089 };
1090
1091 // Adjust the finalization stack, verify the adjustment, and call the
1092 // finalize function a last time to finalize values between the pre-fini
1093 // block and the exit block if we left the parallel "the normal way".
1094 auto FiniInfo = FinalizationStack.pop_back_val();
1095 (void)FiniInfo;
1096 assert(FiniInfo.DK == OMPD_parallel &&(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1097, __extension__
__PRETTY_FUNCTION__))
1097 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_parallel &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_parallel && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1097, __extension__
__PRETTY_FUNCTION__))
;
1098
1099 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
1100
1101 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
1102 FiniCB(PreFiniIP);
1103
1104 OI.OuterAllocaBB = OuterAllocaBlock;
1105 OI.EntryBB = PRegEntryBB;
1106 OI.ExitBB = PRegExitBB;
1107
1108 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
1109 SmallVector<BasicBlock *, 32> Blocks;
1110 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
1111
1112 // Ensure a single exit node for the outlined region by creating one.
1113 // We might have multiple incoming edges to the exit now due to finalizations,
1114 // e.g., cancel calls that cause the control flow to leave the region.
1115 BasicBlock *PRegOutlinedExitBB = PRegExitBB;
1116 PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
1117 PRegOutlinedExitBB->setName("omp.par.outlined.exit");
1118 Blocks.push_back(PRegOutlinedExitBB);
1119
1120 CodeExtractorAnalysisCache CEAC(*OuterFn);
1121 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
1122 /* AggregateArgs */ false,
1123 /* BlockFrequencyInfo */ nullptr,
1124 /* BranchProbabilityInfo */ nullptr,
1125 /* AssumptionCache */ nullptr,
1126 /* AllowVarArgs */ true,
1127 /* AllowAlloca */ true,
1128 /* AllocationBlock */ OuterAllocaBlock,
1129 /* Suffix */ ".omp_par");
1130
1131 // Find inputs to, outputs from the code region.
1132 BasicBlock *CommonExit = nullptr;
1133 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
1134 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1135 Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);
1136
1137 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Before privatization: "
<< *OuterFn << "\n"; } } while (false)
;
1138
1139 FunctionCallee TIDRTLFn =
1140 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
1141
1142 auto PrivHelper = [&](Value &V) {
1143 if (&V == TIDAddr || &V == ZeroAddr) {
1144 OI.ExcludeArgsFromAggregate.push_back(&V);
1145 return;
1146 }
1147
1148 SetVector<Use *> Uses;
1149 for (Use &U : V.uses())
1150 if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
1151 if (ParallelRegionBlockSet.count(UserI->getParent()))
1152 Uses.insert(&U);
1153
1154 // __kmpc_fork_call expects extra arguments as pointers. If the input
1155 // already has a pointer type, everything is fine. Otherwise, store the
1156 // value onto stack and load it back inside the to-be-outlined region. This
1157 // will ensure only the pointer will be passed to the function.
1158 // FIXME: if there are more than 15 trailing arguments, they must be
1159 // additionally packed in a struct.
1160 Value *Inner = &V;
1161 if (!V.getType()->isPointerTy()) {
1162 IRBuilder<>::InsertPointGuard Guard(Builder);
1163 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { llvm::dbgs() << "Forwarding input as pointer: "
<< V << "\n"; } } while (false)
;
1164
1165 Builder.restoreIP(OuterAllocaIP);
1166 Value *Ptr =
1167 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
1168
1169 // Store to stack at end of the block that currently branches to the entry
1170 // block of the to-be-outlined region.
1171 Builder.SetInsertPoint(InsertBB,
1172 InsertBB->getTerminator()->getIterator());
1173 Builder.CreateStore(&V, Ptr);
1174
1175 // Load back next to allocations in the to-be-outlined region.
1176 Builder.restoreIP(InnerAllocaIP);
1177 Inner = Builder.CreateLoad(V.getType(), Ptr);
1178 }
1179
1180 Value *ReplacementValue = nullptr;
1181 CallInst *CI = dyn_cast<CallInst>(&V);
1182 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
1183 ReplacementValue = PrivTID;
1184 } else {
1185 Builder.restoreIP(
1186 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
1187 assert(ReplacementValue &&(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1188, __extension__
__PRETTY_FUNCTION__))
1188 "Expected copy/create callback to set replacement value!")(static_cast <bool> (ReplacementValue && "Expected copy/create callback to set replacement value!"
) ? void (0) : __assert_fail ("ReplacementValue && \"Expected copy/create callback to set replacement value!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1188, __extension__
__PRETTY_FUNCTION__))
;
1189 if (ReplacementValue == &V)
1190 return;
1191 }
1192
1193 for (Use *UPtr : Uses)
1194 UPtr->set(ReplacementValue);
1195 };
1196
1197 // Reset the inner alloca insertion as it will be used for loading the values
1198 // wrapped into pointers before passing them into the to-be-outlined region.
1199 // Configure it to insert immediately after the fake use of zero address so
1200 // that they are available in the generated body and so that the
1201 // OpenMP-related values (thread ID and zero address pointers) remain leading
1202 // in the argument list.
1203 InnerAllocaIP = IRBuilder<>::InsertPoint(
1204 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
1205
1206 // Reset the outer alloca insertion point to the entry of the relevant block
1207 // in case it was invalidated.
1208 OuterAllocaIP = IRBuilder<>::InsertPoint(
1209 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
1210
1211 for (Value *Input : Inputs) {
1212 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Captured input: " <<
*Input << "\n"; } } while (false)
;
1213 PrivHelper(*Input);
1214 }
1215 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1216 for (Value *Output : Outputs)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1217 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
1218 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (Value *Output : Outputs) do {
if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(
"openmp-ir-builder")) { dbgs() << "Captured output: " <<
*Output << "\n"; } } while (false); }; } } while (false
)
;
1219 assert(Outputs.empty() &&(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1220, __extension__
__PRETTY_FUNCTION__))
1220 "OpenMP outlining should not produce live-out values!")(static_cast <bool> (Outputs.empty() && "OpenMP outlining should not produce live-out values!"
) ? void (0) : __assert_fail ("Outputs.empty() && \"OpenMP outlining should not produce live-out values!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1220, __extension__
__PRETTY_FUNCTION__))
;
1221
1222 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "After privatization: "
<< *OuterFn << "\n"; } } while (false)
;
1223 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1224 for (auto *BB : Blocks)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1225 dbgs() << " PBR: " << BB->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
1226 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { for (auto *BB : Blocks) dbgs() <<
" PBR: " << BB->getName() << "\n"; }; } } while
(false)
;
1227
1228 // Register the outlined info.
1229 addOutlineInfo(std::move(OI));
1230
1231 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
1232 UI->eraseFromParent();
1233
1234 return AfterIP;
1235}
1236
1237void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
1238 // Build call void __kmpc_flush(ident_t *loc)
1239 uint32_t SrcLocStrSize;
1240 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1241 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
1242
1243 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
1244}
1245
1246void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
1247 if (!updateToLocation(Loc))
1248 return;
1249 emitFlush(Loc);
1250}
1251
1252void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
1253 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
1254 // global_tid);
1255 uint32_t SrcLocStrSize;
1256 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1257 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1258 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
1259
1260 // Ignore return result until untied tasks are supported.
1261 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
1262 Args);
1263}
1264
1265void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
1266 if (!updateToLocation(Loc))
1267 return;
1268 emitTaskwaitImpl(Loc);
1269}
1270
1271void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
1272 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
1273 uint32_t SrcLocStrSize;
1274 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1275 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1276 Constant *I32Null = ConstantInt::getNullValue(Int32);
1277 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
1278
1279 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
1280 Args);
1281}
1282
1283void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
1284 if (!updateToLocation(Loc))
1285 return;
1286 emitTaskyieldImpl(Loc);
1287}
1288
1289OpenMPIRBuilder::InsertPointTy
1290OpenMPIRBuilder::createTask(const LocationDescription &Loc,
1291 InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
1292 bool Tied, Value *Final) {
1293 if (!updateToLocation(Loc))
1294 return InsertPointTy();
1295
1296 uint32_t SrcLocStrSize;
1297 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1298 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1299 // The current basic block is split into four basic blocks. After outlining,
1300 // they will be mapped as follows:
1301 // ```
1302 // def current_fn() {
1303 // current_basic_block:
1304 // br label %task.exit
1305 // task.exit:
1306 // ; instructions after task
1307 // }
1308 // def outlined_fn() {
1309 // task.alloca:
1310 // br label %task.body
1311 // task.body:
1312 // ret void
1313 // }
1314 // ```
1315 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit");
1316 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body");
1317 BasicBlock *TaskAllocaBB =
1318 splitBB(Builder, /*CreateBranch=*/true, "task.alloca");
1319
1320 OutlineInfo OI;
1321 OI.EntryBB = TaskAllocaBB;
1322 OI.OuterAllocaBB = AllocaIP.getBlock();
1323 OI.ExitBB = TaskExitBB;
1324 OI.PostOutlineCB = [this, Ident, Tied, Final](Function &OutlinedFn) {
1325 // The input IR here looks like the following-
1326 // ```
1327 // func @current_fn() {
1328 // outlined_fn(%args)
1329 // }
1330 // func @outlined_fn(%args) { ... }
1331 // ```
1332 //
1333 // This is changed to the following-
1334 //
1335 // ```
1336 // func @current_fn() {
1337 // runtime_call(..., wrapper_fn, ...)
1338 // }
1339 // func @wrapper_fn(..., %args) {
1340 // outlined_fn(%args)
1341 // }
1342 // func @outlined_fn(%args) { ... }
1343 // ```
1344
1345 // The stale call instruction will be replaced with a new call instruction
1346 // for runtime call with a wrapper function.
1347 assert(OutlinedFn.getNumUses() == 1 &&(static_cast <bool> (OutlinedFn.getNumUses() == 1 &&
"there must be a single user for the outlined function") ? void
(0) : __assert_fail ("OutlinedFn.getNumUses() == 1 && \"there must be a single user for the outlined function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1348, __extension__
__PRETTY_FUNCTION__))
1348 "there must be a single user for the outlined function")(static_cast <bool> (OutlinedFn.getNumUses() == 1 &&
"there must be a single user for the outlined function") ? void
(0) : __assert_fail ("OutlinedFn.getNumUses() == 1 && \"there must be a single user for the outlined function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1348, __extension__
__PRETTY_FUNCTION__))
;
1349 CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
1350
1351 // HasTaskData is true if any variables are captured in the outlined region,
1352 // false otherwise.
1353 bool HasTaskData = StaleCI->arg_size() > 0;
1354 Builder.SetInsertPoint(StaleCI);
1355
1356 // Gather the arguments for emitting the runtime call for
1357 // @__kmpc_omp_task_alloc
1358 Function *TaskAllocFn =
1359 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
1360
1361 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
1362 // call.
1363 Value *ThreadID = getOrCreateThreadID(Ident);
1364
1365 // Argument - `flags`
1366 // Task is tied iff (Flags & 1) == 1.
1367 // Task is untied iff (Flags & 1) == 0.
1368 // Task is final iff (Flags & 2) == 2.
1369 // Task is not final iff (Flags & 2) == 0.
1370 // TODO: Handle the other flags.
1371 Value *Flags = Builder.getInt32(Tied);
1372 if (Final) {
1373 Value *FinalFlag =
1374 Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0));
1375 Flags = Builder.CreateOr(FinalFlag, Flags);
1376 }
1377
1378 // Argument - `sizeof_kmp_task_t` (TaskSize)
1379 // Tasksize refers to the size in bytes of kmp_task_t data structure
1380 // including private vars accessed in task.
1381 Value *TaskSize = Builder.getInt64(0);
1382 if (HasTaskData) {
1383 AllocaInst *ArgStructAlloca =
1384 dyn_cast<AllocaInst>(StaleCI->getArgOperand(0));
1385 assert(ArgStructAlloca &&(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1387, __extension__
__PRETTY_FUNCTION__))
1386 "Unable to find the alloca instruction corresponding to arguments "(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1387, __extension__
__PRETTY_FUNCTION__))
1387 "for extracted function")(static_cast <bool> (ArgStructAlloca && "Unable to find the alloca instruction corresponding to arguments "
"for extracted function") ? void (0) : __assert_fail ("ArgStructAlloca && \"Unable to find the alloca instruction corresponding to arguments \" \"for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1387, __extension__
__PRETTY_FUNCTION__))
;
1388 StructType *ArgStructType =
1389 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
1390 assert(ArgStructType && "Unable to find struct type corresponding to "(static_cast <bool> (ArgStructType && "Unable to find struct type corresponding to "
"arguments for extracted function") ? void (0) : __assert_fail
("ArgStructType && \"Unable to find struct type corresponding to \" \"arguments for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1391, __extension__
__PRETTY_FUNCTION__))
1391 "arguments for extracted function")(static_cast <bool> (ArgStructType && "Unable to find struct type corresponding to "
"arguments for extracted function") ? void (0) : __assert_fail
("ArgStructType && \"Unable to find struct type corresponding to \" \"arguments for extracted function\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1391, __extension__
__PRETTY_FUNCTION__))
;
1392 TaskSize =
1393 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
1394 }
1395
1396 // TODO: Argument - sizeof_shareds
1397
1398 // Argument - task_entry (the wrapper function)
1399 // If the outlined function has some captured variables (i.e. HasTaskData is
1400 // true), then the wrapper function will have an additional argument (the
1401 // struct containing captured variables). Otherwise, no such argument will
1402 // be present.
1403 SmallVector<Type *> WrapperArgTys{Builder.getInt32Ty()};
1404 if (HasTaskData)
1405 WrapperArgTys.push_back(OutlinedFn.getArg(0)->getType());
1406 FunctionCallee WrapperFuncVal = M.getOrInsertFunction(
1407 (Twine(OutlinedFn.getName()) + ".wrapper").str(),
1408 FunctionType::get(Builder.getInt32Ty(), WrapperArgTys, false));
1409 Function *WrapperFunc = dyn_cast<Function>(WrapperFuncVal.getCallee());
1410 PointerType *WrapperFuncBitcastType =
1411 FunctionType::get(Builder.getInt32Ty(),
1412 {Builder.getInt32Ty(), Builder.getInt8PtrTy()}, false)
1413 ->getPointerTo();
1414 Value *WrapperFuncBitcast =
1415 ConstantExpr::getBitCast(WrapperFunc, WrapperFuncBitcastType);
1416
1417 // Emit the @__kmpc_omp_task_alloc runtime call
1418 // The runtime call returns a pointer to an area where the task captured
1419 // variables must be copied before the task is run (NewTaskData)
1420 CallInst *NewTaskData = Builder.CreateCall(
1421 TaskAllocFn,
1422 {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
1423 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/Builder.getInt64(0),
1424 /*task_func=*/WrapperFuncBitcast});
1425
1426 // Copy the arguments for outlined function
1427 if (HasTaskData) {
1428 Value *TaskData = StaleCI->getArgOperand(0);
1429 Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());
1430 Builder.CreateMemCpy(NewTaskData, Alignment, TaskData, Alignment,
1431 TaskSize);
1432 }
1433
1434 // Emit the @__kmpc_omp_task runtime call to spawn the task
1435 Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);
1436 Builder.CreateCall(TaskFn, {Ident, ThreadID, NewTaskData});
1437
1438 StaleCI->eraseFromParent();
1439
1440 // Emit the body for wrapper function
1441 BasicBlock *WrapperEntryBB =
1442 BasicBlock::Create(M.getContext(), "", WrapperFunc);
1443 Builder.SetInsertPoint(WrapperEntryBB);
1444 if (HasTaskData)
1445 Builder.CreateCall(&OutlinedFn, {WrapperFunc->getArg(1)});
1446 else
1447 Builder.CreateCall(&OutlinedFn);
1448 Builder.CreateRet(Builder.getInt32(0));
1449 };
1450
1451 addOutlineInfo(std::move(OI));
1452
1453 InsertPointTy TaskAllocaIP =
1454 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
1455 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
1456 BodyGenCB(TaskAllocaIP, TaskBodyIP);
1457 Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin());
1458
1459 return Builder.saveIP();
1460}
1461
1462OpenMPIRBuilder::InsertPointTy
1463OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
1464 InsertPointTy AllocaIP,
1465 BodyGenCallbackTy BodyGenCB) {
1466 if (!updateToLocation(Loc))
1467 return InsertPointTy();
1468
1469 uint32_t SrcLocStrSize;
1470 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1471 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1472 Value *ThreadID = getOrCreateThreadID(Ident);
1473
1474 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
1475 Function *TaskgroupFn =
1476 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup);
1477 Builder.CreateCall(TaskgroupFn, {Ident, ThreadID});
1478
1479 BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
1480 BodyGenCB(AllocaIP, Builder.saveIP());
1481
1482 Builder.SetInsertPoint(TaskgroupExitBB);
1483 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
1484 Function *EndTaskgroupFn =
1485 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup);
1486 Builder.CreateCall(EndTaskgroupFn, {Ident, ThreadID});
1487
1488 return Builder.saveIP();
1489}
1490
1491OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
1492 const LocationDescription &Loc, InsertPointTy AllocaIP,
1493 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
1494 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
1495 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required")(static_cast <bool> (!isConflictIP(AllocaIP, Loc.IP) &&
"Dedicated IP allocas required") ? void (0) : __assert_fail (
"!isConflictIP(AllocaIP, Loc.IP) && \"Dedicated IP allocas required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1495, __extension__
__PRETTY_FUNCTION__))
;
1496
1497 if (!updateToLocation(Loc))
1498 return Loc.IP;
1499
1500 auto FiniCBWrapper = [&](InsertPointTy IP) {
1501 if (IP.getBlock()->end() != IP.getPoint())
1502 return FiniCB(IP);
1503 // This must be done otherwise any nested constructs using FinalizeOMPRegion
1504 // will fail because that function requires the Finalization Basic Block to
1505 // have a terminator, which is already removed by EmitOMPRegionBody.
1506 // IP is currently at cancelation block.
1507 // We need to backtrack to the condition block to fetch
1508 // the exit block and create a branch from cancelation
1509 // to exit block.
1510 IRBuilder<>::InsertPointGuard IPG(Builder);
1511 Builder.restoreIP(IP);
1512 auto *CaseBB = IP.getBlock()->getSinglePredecessor();
1513 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1514 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
1515 Instruction *I = Builder.CreateBr(ExitBB);
1516 IP = InsertPointTy(I->getParent(), I->getIterator());
1517 return FiniCB(IP);
1518 };
1519
1520 FinalizationStack.push_back({FiniCBWrapper, OMPD_sections, IsCancellable});
1521
1522 // Each section is emitted as a switch case
1523 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
1524 // -> OMP.createSection() which generates the IR for each section
1525 // Iterate through all sections and emit a switch construct:
1526 // switch (IV) {
1527 // case 0:
1528 // <SectionStmt[0]>;
1529 // break;
1530 // ...
1531 // case <NumSection> - 1:
1532 // <SectionStmt[<NumSection> - 1]>;
1533 // break;
1534 // }
1535 // ...
1536 // section_loop.after:
1537 // <FiniCB>;
1538 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
1539 Builder.restoreIP(CodeGenIP);
1540 BasicBlock *Continue =
1541 splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");
1542 Function *CurFn = Continue->getParent();
1543 SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue);
1544
1545 unsigned CaseNumber = 0;
1546 for (auto SectionCB : SectionCBs) {
1547 BasicBlock *CaseBB = BasicBlock::Create(
1548 M.getContext(), "omp_section_loop.body.case", CurFn, Continue);
1549 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
1550 Builder.SetInsertPoint(CaseBB);
1551 BranchInst *CaseEndBr = Builder.CreateBr(Continue);
1552 SectionCB(InsertPointTy(),
1553 {CaseEndBr->getParent(), CaseEndBr->getIterator()});
1554 CaseNumber++;
1555 }
1556 // remove the existing terminator from body BB since there can be no
1557 // terminators after switch/case
1558 };
1559 // Loop body ends here
1560 // LowerBound, UpperBound, and STride for createCanonicalLoop
1561 Type *I32Ty = Type::getInt32Ty(M.getContext());
1562 Value *LB = ConstantInt::get(I32Ty, 0);
1563 Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());
1564 Value *ST = ConstantInt::get(I32Ty, 1);
1565 llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
1566 Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");
1567 InsertPointTy AfterIP =
1568 applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);
1569
1570 // Apply the finalization callback in LoopAfterBB
1571 auto FiniInfo = FinalizationStack.pop_back_val();
1572 assert(FiniInfo.DK == OMPD_sections &&(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1573, __extension__
__PRETTY_FUNCTION__))
1573 "Unexpected finalization stack state!")(static_cast <bool> (FiniInfo.DK == OMPD_sections &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("FiniInfo.DK == OMPD_sections && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1573, __extension__
__PRETTY_FUNCTION__))
;
1574 if (FinalizeCallbackTy &CB = FiniInfo.FiniCB) {
1575 Builder.restoreIP(AfterIP);
1576 BasicBlock *FiniBB =
1577 splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini");
1578 CB(Builder.saveIP());
1579 AfterIP = {FiniBB, FiniBB->begin()};
1580 }
1581
1582 return AfterIP;
1583}
1584
1585OpenMPIRBuilder::InsertPointTy
1586OpenMPIRBuilder::createSection(const LocationDescription &Loc,
1587 BodyGenCallbackTy BodyGenCB,
1588 FinalizeCallbackTy FiniCB) {
1589 if (!updateToLocation(Loc))
1590 return Loc.IP;
1591
1592 auto FiniCBWrapper = [&](InsertPointTy IP) {
1593 if (IP.getBlock()->end() != IP.getPoint())
1594 return FiniCB(IP);
1595 // This must be done otherwise any nested constructs using FinalizeOMPRegion
1596 // will fail because that function requires the Finalization Basic Block to
1597 // have a terminator, which is already removed by EmitOMPRegionBody.
1598 // IP is currently at cancelation block.
1599 // We need to backtrack to the condition block to fetch
1600 // the exit block and create a branch from cancelation
1601 // to exit block.
1602 IRBuilder<>::InsertPointGuard IPG(Builder);
1603 Builder.restoreIP(IP);
1604 auto *CaseBB = Loc.IP.getBlock();
1605 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1606 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
1607 Instruction *I = Builder.CreateBr(ExitBB);
1608 IP = InsertPointTy(I->getParent(), I->getIterator());
1609 return FiniCB(IP);
1610 };
1611
1612 Directive OMPD = Directive::OMPD_sections;
1613 // Since we are using Finalization Callback here, HasFinalize
1614 // and IsCancellable have to be true
1615 return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,
1616 /*Conditional*/ false, /*hasFinalize*/ true,
1617 /*IsCancellable*/ true);
1618}
1619
1620/// Create a function with a unique name and a "void (i8*, i8*)" signature in
1621/// the given module and return it.
1622Function *getFreshReductionFunc(Module &M) {
1623 Type *VoidTy = Type::getVoidTy(M.getContext());
1624 Type *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
1625 auto *FuncTy =
1626 FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
1627 return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
1628 M.getDataLayout().getDefaultGlobalsAddressSpace(),
1629 ".omp.reduction.func", &M);
1630}
1631
1632OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
1633 const LocationDescription &Loc, InsertPointTy AllocaIP,
1634 ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait) {
1635 for (const ReductionInfo &RI : ReductionInfos) {
1636 (void)RI;
1637 assert(RI.Variable && "expected non-null variable")(static_cast <bool> (RI.Variable && "expected non-null variable"
) ? void (0) : __assert_fail ("RI.Variable && \"expected non-null variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1637, __extension__
__PRETTY_FUNCTION__))
;
1638 assert(RI.PrivateVariable && "expected non-null private variable")(static_cast <bool> (RI.PrivateVariable && "expected non-null private variable"
) ? void (0) : __assert_fail ("RI.PrivateVariable && \"expected non-null private variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1638, __extension__
__PRETTY_FUNCTION__))
;
1639 assert(RI.ReductionGen && "expected non-null reduction generator callback")(static_cast <bool> (RI.ReductionGen && "expected non-null reduction generator callback"
) ? void (0) : __assert_fail ("RI.ReductionGen && \"expected non-null reduction generator callback\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1639, __extension__
__PRETTY_FUNCTION__))
;
1640 assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
1641 "expected variables and their private equivalents to have the same "(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
1642 "type")(static_cast <bool> (RI.Variable->getType() == RI.PrivateVariable
->getType() && "expected variables and their private equivalents to have the same "
"type") ? void (0) : __assert_fail ("RI.Variable->getType() == RI.PrivateVariable->getType() && \"expected variables and their private equivalents to have the same \" \"type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
;
1643 assert(RI.Variable->getType()->isPointerTy() &&(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1644, __extension__
__PRETTY_FUNCTION__))
1644 "expected variables to be pointers")(static_cast <bool> (RI.Variable->getType()->isPointerTy
() && "expected variables to be pointers") ? void (0)
: __assert_fail ("RI.Variable->getType()->isPointerTy() && \"expected variables to be pointers\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1644, __extension__
__PRETTY_FUNCTION__))
;
1645 }
1646
1647 if (!updateToLocation(Loc))
1648 return InsertPointTy();
1649
1650 BasicBlock *InsertBlock = Loc.IP.getBlock();
1651 BasicBlock *ContinuationBlock =
1652 InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
1653 InsertBlock->getTerminator()->eraseFromParent();
1654
1655 // Create and populate array of type-erased pointers to private reduction
1656 // values.
1657 unsigned NumReductions = ReductionInfos.size();
1658 Type *RedArrayTy = ArrayType::get(Builder.getInt8PtrTy(), NumReductions);
1659 Builder.restoreIP(AllocaIP);
1660 Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");
1661
1662 Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
1663
1664 for (auto En : enumerate(ReductionInfos)) {
1665 unsigned Index = En.index();
1666 const ReductionInfo &RI = En.value();
1667 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
1668 RedArrayTy, RedArray, 0, Index, "red.array.elem." + Twine(Index));
1669 Value *Casted =
1670 Builder.CreateBitCast(RI.PrivateVariable, Builder.getInt8PtrTy(),
1671 "private.red.var." + Twine(Index) + ".casted");
1672 Builder.CreateStore(Casted, RedArrayElemPtr);
1673 }
1674
1675 // Emit a call to the runtime function that orchestrates the reduction.
1676 // Declare the reduction function in the process.
1677 Function *Func = Builder.GetInsertBlock()->getParent();
1678 Module *Module = Func->getParent();
1679 Value *RedArrayPtr =
1680 Builder.CreateBitCast(RedArray, Builder.getInt8PtrTy(), "red.array.ptr");
1681 uint32_t SrcLocStrSize;
1682 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1683 bool CanGenerateAtomic =
1684 llvm::all_of(ReductionInfos, [](const ReductionInfo &RI) {
1685 return RI.AtomicReductionGen;
1686 });
1687 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
1688 CanGenerateAtomic
1689 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
1690 : IdentFlag(0));
1691 Value *ThreadId = getOrCreateThreadID(Ident);
1692 Constant *NumVariables = Builder.getInt32(NumReductions);
1693 const DataLayout &DL = Module->getDataLayout();
1694 unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
1695 Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
1696 Function *ReductionFunc = getFreshReductionFunc(*Module);
1697 Value *Lock = getOMPCriticalRegionLock(".reduction");
1698 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
1699 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
1700 : RuntimeFunction::OMPRTL___kmpc_reduce);
1701 CallInst *ReduceCall =
1702 Builder.CreateCall(ReduceFunc,
1703 {Ident, ThreadId, NumVariables, RedArraySize,
1704 RedArrayPtr, ReductionFunc, Lock},
1705 "reduce");
1706
1707 // Create final reduction entry blocks for the atomic and non-atomic case.
1708 // Emit IR that dispatches control flow to one of the blocks based on the
1709 // reduction supporting the atomic mode.
1710 BasicBlock *NonAtomicRedBlock =
1711 BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func);
1712 BasicBlock *AtomicRedBlock =
1713 BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func);
1714 SwitchInst *Switch =
1715 Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);
1716 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);
1717 Switch->addCase(Builder.getInt32(2), AtomicRedBlock);
1718
1719 // Populate the non-atomic reduction using the elementwise reduction function.
1720 // This loads the elements from the global and private variables and reduces
1721 // them before storing back the result to the global variable.
1722 Builder.SetInsertPoint(NonAtomicRedBlock);
1723 for (auto En : enumerate(ReductionInfos)) {
1724 const ReductionInfo &RI = En.value();
1725 Type *ValueType = RI.ElementType;
1726 Value *RedValue = Builder.CreateLoad(ValueType, RI.Variable,
1727 "red.value." + Twine(En.index()));
1728 Value *PrivateRedValue =
1729 Builder.CreateLoad(ValueType, RI.PrivateVariable,
1730 "red.private.value." + Twine(En.index()));
1731 Value *Reduced;
1732 Builder.restoreIP(
1733 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced));
1734 if (!Builder.GetInsertBlock())
1735 return InsertPointTy();
1736 Builder.CreateStore(Reduced, RI.Variable);
1737 }
1738 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
1739 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
1740 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
1741 Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock});
1742 Builder.CreateBr(ContinuationBlock);
1743
1744 // Populate the atomic reduction using the atomic elementwise reduction
1745 // function. There are no loads/stores here because they will be happening
1746 // inside the atomic elementwise reduction.
1747 Builder.SetInsertPoint(AtomicRedBlock);
1748 if (CanGenerateAtomic) {
1749 for (const ReductionInfo &RI : ReductionInfos) {
1750 Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
1751 RI.Variable, RI.PrivateVariable));
1752 if (!Builder.GetInsertBlock())
1753 return InsertPointTy();
1754 }
1755 Builder.CreateBr(ContinuationBlock);
1756 } else {
1757 Builder.CreateUnreachable();
1758 }
1759
1760 // Populate the outlined reduction function using the elementwise reduction
1761 // function. Partial values are extracted from the type-erased array of
1762 // pointers to private variables.
1763 BasicBlock *ReductionFuncBlock =
1764 BasicBlock::Create(Module->getContext(), "", ReductionFunc);
1765 Builder.SetInsertPoint(ReductionFuncBlock);
1766 Value *LHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(0),
1767 RedArrayTy->getPointerTo());
1768 Value *RHSArrayPtr = Builder.CreateBitCast(ReductionFunc->getArg(1),
1769 RedArrayTy->getPointerTo());
1770 for (auto En : enumerate(ReductionInfos)) {
1771 const ReductionInfo &RI = En.value();
1772 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1773 RedArrayTy, LHSArrayPtr, 0, En.index());
1774 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), LHSI8PtrPtr);
1775 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
1776 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
1777 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
1778 RedArrayTy, RHSArrayPtr, 0, En.index());
1779 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), RHSI8PtrPtr);
1780 Value *RHSPtr =
1781 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
1782 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
1783 Value *Reduced;
1784 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
1785 if (!Builder.GetInsertBlock())
1786 return InsertPointTy();
1787 Builder.CreateStore(Reduced, LHSPtr);
1788 }
1789 Builder.CreateRetVoid();
1790
1791 Builder.SetInsertPoint(ContinuationBlock);
1792 return Builder.saveIP();
1793}
1794
1795OpenMPIRBuilder::InsertPointTy
1796OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
1797 BodyGenCallbackTy BodyGenCB,
1798 FinalizeCallbackTy FiniCB) {
1799
1800 if (!updateToLocation(Loc))
1801 return Loc.IP;
1802
1803 Directive OMPD = Directive::OMPD_master;
1804 uint32_t SrcLocStrSize;
1805 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1806 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1807 Value *ThreadId = getOrCreateThreadID(Ident);
1808 Value *Args[] = {Ident, ThreadId};
1809
1810 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
1811 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1812
1813 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
1814 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1815
1816 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1817 /*Conditional*/ true, /*hasFinalize*/ true);
1818}
1819
1820OpenMPIRBuilder::InsertPointTy
1821OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
1822 BodyGenCallbackTy BodyGenCB,
1823 FinalizeCallbackTy FiniCB, Value *Filter) {
1824 if (!updateToLocation(Loc))
1825 return Loc.IP;
1826
1827 Directive OMPD = Directive::OMPD_masked;
1828 uint32_t SrcLocStrSize;
1829 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1830 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1831 Value *ThreadId = getOrCreateThreadID(Ident);
1832 Value *Args[] = {Ident, ThreadId, Filter};
1833 Value *ArgsEnd[] = {Ident, ThreadId};
1834
1835 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_masked);
1836 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1837
1838 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_masked);
1839 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd);
1840
1841 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1842 /*Conditional*/ true, /*hasFinalize*/ true);
1843}
1844
1845CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
1846 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
1847 BasicBlock *PostInsertBefore, const Twine &Name) {
1848 Module *M = F->getParent();
1849 LLVMContext &Ctx = M->getContext();
1850 Type *IndVarTy = TripCount->getType();
1851
1852 // Create the basic block structure.
1853 BasicBlock *Preheader =
1854 BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
1855 BasicBlock *Header =
1856 BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
1857 BasicBlock *Cond =
1858 BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
1859 BasicBlock *Body =
1860 BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
1861 BasicBlock *Latch =
1862 BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
1863 BasicBlock *Exit =
1864 BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
1865 BasicBlock *After =
1866 BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
1867
1868 // Use specified DebugLoc for new instructions.
1869 Builder.SetCurrentDebugLocation(DL);
1870
1871 Builder.SetInsertPoint(Preheader);
1872 Builder.CreateBr(Header);
1873
1874 Builder.SetInsertPoint(Header);
1875 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
1876 IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
1877 Builder.CreateBr(Cond);
1878
1879 Builder.SetInsertPoint(Cond);
1880 Value *Cmp =
1881 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
1882 Builder.CreateCondBr(Cmp, Body, Exit);
1883
1884 Builder.SetInsertPoint(Body);
1885 Builder.CreateBr(Latch);
1886
1887 Builder.SetInsertPoint(Latch);
1888 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
1889 "omp_" + Name + ".next", /*HasNUW=*/true);
1890 Builder.CreateBr(Header);
1891 IndVarPHI->addIncoming(Next, Latch);
1892
1893 Builder.SetInsertPoint(Exit);
1894 Builder.CreateBr(After);
1895
1896 // Remember and return the canonical control flow.
1897 LoopInfos.emplace_front();
1898 CanonicalLoopInfo *CL = &LoopInfos.front();
1899
1900 CL->Header = Header;
1901 CL->Cond = Cond;
1902 CL->Latch = Latch;
1903 CL->Exit = Exit;
1904
1905#ifndef NDEBUG
1906 CL->assertOK();
1907#endif
1908 return CL;
1909}
1910
1911CanonicalLoopInfo *
1912OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
1913 LoopBodyGenCallbackTy BodyGenCB,
1914 Value *TripCount, const Twine &Name) {
1915 BasicBlock *BB = Loc.IP.getBlock();
1916 BasicBlock *NextBB = BB->getNextNode();
1917
1918 CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
1919 NextBB, NextBB, Name);
1920 BasicBlock *After = CL->getAfter();
1921
1922 // If location is not set, don't connect the loop.
1923 if (updateToLocation(Loc)) {
1924 // Split the loop at the insertion point: Branch to the preheader and move
1925 // every following instruction to after the loop (the After BB). Also, the
1926 // new successor is the loop's after block.
1927 spliceBB(Builder, After, /*CreateBranch=*/false);
1928 Builder.CreateBr(CL->getPreheader());
1929 }
1930
1931 // Emit the body content. We do it after connecting the loop to the CFG to
1932 // avoid that the callback encounters degenerate BBs.
1933 BodyGenCB(CL->getBodyIP(), CL->getIndVar());
1934
1935#ifndef NDEBUG
1936 CL->assertOK();
1937#endif
1938 return CL;
1939}
1940
1941CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
1942 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
1943 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
1944 InsertPointTy ComputeIP, const Twine &Name) {
1945
1946 // Consider the following difficulties (assuming 8-bit signed integers):
1947 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
1948 // DO I = 1, 100, 50
1949 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
1950 // DO I = 100, 0, -128
1951
1952 // Start, Stop and Step must be of the same integer type.
1953 auto *IndVarTy = cast<IntegerType>(Start->getType());
1954 assert(IndVarTy == Stop->getType() && "Stop type mismatch")(static_cast <bool> (IndVarTy == Stop->getType() &&
"Stop type mismatch") ? void (0) : __assert_fail ("IndVarTy == Stop->getType() && \"Stop type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1954, __extension__
__PRETTY_FUNCTION__))
;
1955 assert(IndVarTy == Step->getType() && "Step type mismatch")(static_cast <bool> (IndVarTy == Step->getType() &&
"Step type mismatch") ? void (0) : __assert_fail ("IndVarTy == Step->getType() && \"Step type mismatch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 1955, __extension__
__PRETTY_FUNCTION__))
;
1956
1957 LocationDescription ComputeLoc =
1958 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
1959 updateToLocation(ComputeLoc);
1960
1961 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
1962 ConstantInt *One = ConstantInt::get(IndVarTy, 1);
1963
1964 // Like Step, but always positive.
1965 Value *Incr = Step;
1966
1967 // Distance between Start and Stop; always positive.
1968 Value *Span;
1969
1970 // Condition whether there are no iterations are executed at all, e.g. because
1971 // UB < LB.
1972 Value *ZeroCmp;
1973
1974 if (IsSigned) {
1975 // Ensure that increment is positive. If not, negate and invert LB and UB.
1976 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
1977 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
1978 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
1979 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
1980 Span = Builder.CreateSub(UB, LB, "", false, true);
1981 ZeroCmp = Builder.CreateICmp(
1982 InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
1983 } else {
1984 Span = Builder.CreateSub(Stop, Start, "", true);
1985 ZeroCmp = Builder.CreateICmp(
1986 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
1987 }
1988
1989 Value *CountIfLooping;
1990 if (InclusiveStop) {
1991 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
1992 } else {
1993 // Avoid incrementing past stop since it could overflow.
1994 Value *CountIfTwo = Builder.CreateAdd(
1995 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
1996 Value *OneCmp = Builder.CreateICmp(
1997 InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
1998 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
1999 }
2000 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
2001 "omp_" + Name + ".tripcount");
2002
2003 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
2004 Builder.restoreIP(CodeGenIP);
2005 Value *Span = Builder.CreateMul(IV, Step);
2006 Value *IndVar = Builder.CreateAdd(Span, Start);
2007 BodyGenCB(Builder.saveIP(), IndVar);
2008 };
2009 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
2010 return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
2011}
2012
2013// Returns an LLVM function to call for initializing loop bounds using OpenMP
2014// static scheduling depending on `type`. Only i32 and i64 are supported by the
2015// runtime. Always interpret integers as unsigned similarly to
2016// CanonicalLoopInfo.
2017static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
2018 OpenMPIRBuilder &OMPBuilder) {
2019 unsigned Bitwidth = Ty->getIntegerBitWidth();
2020 if (Bitwidth == 32)
2021 return OMPBuilder.getOrCreateRuntimeFunction(
2022 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
2023 if (Bitwidth == 64)
2024 return OMPBuilder.getOrCreateRuntimeFunction(
2025 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
2026 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2026)
;
2027}
2028
2029OpenMPIRBuilder::InsertPointTy
2030OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
2031 InsertPointTy AllocaIP,
2032 bool NeedsBarrier) {
2033 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2033, __extension__
__PRETTY_FUNCTION__))
;
2034 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2035, __extension__
__PRETTY_FUNCTION__))
2035 "Require dedicated allocate IP")(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2035, __extension__
__PRETTY_FUNCTION__))
;
2036
2037 // Set up the source location value for OpenMP runtime.
2038 Builder.restoreIP(CLI->getPreheaderIP());
2039 Builder.SetCurrentDebugLocation(DL);
2040
2041 uint32_t SrcLocStrSize;
2042 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2043 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2044
2045 // Declare useful OpenMP runtime functions.
2046 Value *IV = CLI->getIndVar();
2047 Type *IVTy = IV->getType();
2048 FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
2049 FunctionCallee StaticFini =
2050 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
2051
2052 // Allocate space for computed loop bounds as expected by the "init" function.
2053 Builder.restoreIP(AllocaIP);
2054 Type *I32Type = Type::getInt32Ty(M.getContext());
2055 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2056 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
2057 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
2058 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
2059
2060 // At the end of the preheader, prepare for calling the "init" function by
2061 // storing the current loop bounds into the allocated space. A canonical loop
2062 // always iterates from 0 to trip-count with step 1. Note that "init" expects
2063 // and produces an inclusive upper bound.
2064 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2065 Constant *Zero = ConstantInt::get(IVTy, 0);
2066 Constant *One = ConstantInt::get(IVTy, 1);
2067 Builder.CreateStore(Zero, PLowerBound);
2068 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
2069 Builder.CreateStore(UpperBound, PUpperBound);
2070 Builder.CreateStore(One, PStride);
2071
2072 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2073
2074 Constant *SchedulingType = ConstantInt::get(
2075 I32Type, static_cast<int>(OMPScheduleType::UnorderedStatic));
2076
2077 // Call the "init" function and update the trip count of the loop with the
2078 // value it produced.
2079 Builder.CreateCall(StaticInit,
2080 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
2081 PUpperBound, PStride, One, Zero});
2082 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
2083 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
2084 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
2085 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
2086 CLI->setTripCount(TripCount);
2087
2088 // Update all uses of the induction variable except the one in the condition
2089 // block that compares it with the actual upper bound, and the increment in
2090 // the latch block.
2091
2092 CLI->mapIndVar([&](Instruction *OldIV) -> Value * {
2093 Builder.SetInsertPoint(CLI->getBody(),
2094 CLI->getBody()->getFirstInsertionPt());
2095 Builder.SetCurrentDebugLocation(DL);
2096 return Builder.CreateAdd(OldIV, LowerBound);
2097 });
2098
2099 // In the "exit" block, call the "fini" function.
2100 Builder.SetInsertPoint(CLI->getExit(),
2101 CLI->getExit()->getTerminator()->getIterator());
2102 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
2103
2104 // Add the barrier if requested.
2105 if (NeedsBarrier)
2106 createBarrier(LocationDescription(Builder.saveIP(), DL),
2107 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
2108 /* CheckCancelFlag */ false);
2109
2110 InsertPointTy AfterIP = CLI->getAfterIP();
2111 CLI->invalidate();
2112
2113 return AfterIP;
2114}
2115
2116OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
2117 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2118 bool NeedsBarrier, Value *ChunkSize) {
2119 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2119, __extension__
__PRETTY_FUNCTION__))
;
2120 assert(ChunkSize && "Chunk size is required")(static_cast <bool> (ChunkSize && "Chunk size is required"
) ? void (0) : __assert_fail ("ChunkSize && \"Chunk size is required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2120, __extension__
__PRETTY_FUNCTION__))
;
2121
2122 LLVMContext &Ctx = CLI->getFunction()->getContext();
2123 Value *IV = CLI->getIndVar();
2124 Value *OrigTripCount = CLI->getTripCount();
2125 Type *IVTy = IV->getType();
2126 assert(IVTy->getIntegerBitWidth() <= 64 &&(static_cast <bool> (IVTy->getIntegerBitWidth() <=
64 && "Max supported tripcount bitwidth is 64 bits")
? void (0) : __assert_fail ("IVTy->getIntegerBitWidth() <= 64 && \"Max supported tripcount bitwidth is 64 bits\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2127, __extension__
__PRETTY_FUNCTION__))
2127 "Max supported tripcount bitwidth is 64 bits")(static_cast <bool> (IVTy->getIntegerBitWidth() <=
64 && "Max supported tripcount bitwidth is 64 bits")
? void (0) : __assert_fail ("IVTy->getIntegerBitWidth() <= 64 && \"Max supported tripcount bitwidth is 64 bits\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2127, __extension__
__PRETTY_FUNCTION__))
;
2128 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(Ctx)
2129 : Type::getInt64Ty(Ctx);
2130 Type *I32Type = Type::getInt32Ty(M.getContext());
2131 Constant *Zero = ConstantInt::get(InternalIVTy, 0);
2132 Constant *One = ConstantInt::get(InternalIVTy, 1);
2133
2134 // Declare useful OpenMP runtime functions.
2135 FunctionCallee StaticInit =
2136 getKmpcForStaticInitForType(InternalIVTy, M, *this);
2137 FunctionCallee StaticFini =
2138 getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
2139
2140 // Allocate space for computed loop bounds as expected by the "init" function.
2141 Builder.restoreIP(AllocaIP);
2142 Builder.SetCurrentDebugLocation(DL);
2143 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2144 Value *PLowerBound =
2145 Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound");
2146 Value *PUpperBound =
2147 Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound");
2148 Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride");
2149
2150 // Set up the source location value for the OpenMP runtime.
2151 Builder.restoreIP(CLI->getPreheaderIP());
2152 Builder.SetCurrentDebugLocation(DL);
2153
2154 // TODO: Detect overflow in ubsan or max-out with current tripcount.
2155 Value *CastedChunkSize =
2156 Builder.CreateZExtOrTrunc(ChunkSize, InternalIVTy, "chunksize");
2157 Value *CastedTripCount =
2158 Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount");
2159
2160 Constant *SchedulingType = ConstantInt::get(
2161 I32Type, static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
2162 Builder.CreateStore(Zero, PLowerBound);
2163 Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One);
2164 Builder.CreateStore(OrigUpperBound, PUpperBound);
2165 Builder.CreateStore(One, PStride);
2166
2167 // Call the "init" function and update the trip count of the loop with the
2168 // value it produced.
2169 uint32_t SrcLocStrSize;
2170 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2171 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2172 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2173 Builder.CreateCall(StaticInit,
2174 {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
2175 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
2176 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
2177 /*pstride=*/PStride, /*incr=*/One,
2178 /*chunk=*/CastedChunkSize});
2179
2180 // Load values written by the "init" function.
2181 Value *FirstChunkStart =
2182 Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb");
2183 Value *FirstChunkStop =
2184 Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub");
2185 Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One);
2186 Value *ChunkRange =
2187 Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range");
2188 Value *NextChunkStride =
2189 Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride");
2190
2191 // Create outer "dispatch" loop for enumerating the chunks.
2192 BasicBlock *DispatchEnter = splitBB(Builder, true);
2193 Value *DispatchCounter;
2194 CanonicalLoopInfo *DispatchCLI = createCanonicalLoop(
2195 {Builder.saveIP(), DL},
2196 [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; },
2197 FirstChunkStart, CastedTripCount, NextChunkStride,
2198 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
2199 "dispatch");
2200
2201 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
2202 // not have to preserve the canonical invariant.
2203 BasicBlock *DispatchBody = DispatchCLI->getBody();
2204 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
2205 BasicBlock *DispatchExit = DispatchCLI->getExit();
2206 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
2207 DispatchCLI->invalidate();
2208
2209 // Rewire the original loop to become the chunk loop inside the dispatch loop.
2210 redirectTo(DispatchAfter, CLI->getAfter(), DL);
2211 redirectTo(CLI->getExit(), DispatchLatch, DL);
2212 redirectTo(DispatchBody, DispatchEnter, DL);
2213
2214 // Prepare the prolog of the chunk loop.
2215 Builder.restoreIP(CLI->getPreheaderIP());
2216 Builder.SetCurrentDebugLocation(DL);
2217
2218 // Compute the number of iterations of the chunk loop.
2219 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2220 Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange);
2221 Value *IsLastChunk =
2222 Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last");
2223 Value *CountUntilOrigTripCount =
2224 Builder.CreateSub(CastedTripCount, DispatchCounter);
2225 Value *ChunkTripCount = Builder.CreateSelect(
2226 IsLastChunk, CountUntilOrigTripCount, ChunkRange, "omp_chunk.tripcount");
2227 Value *BackcastedChunkTC =
2228 Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc");
2229 CLI->setTripCount(BackcastedChunkTC);
2230
2231 // Update all uses of the induction variable except the one in the condition
2232 // block that compares it with the actual upper bound, and the increment in
2233 // the latch block.
2234 Value *BackcastedDispatchCounter =
2235 Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc");
2236 CLI->mapIndVar([&](Instruction *) -> Value * {
2237 Builder.restoreIP(CLI->getBodyIP());
2238 return Builder.CreateAdd(IV, BackcastedDispatchCounter);
2239 });
2240
2241 // In the "exit" block, call the "fini" function.
2242 Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt());
2243 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
2244
2245 // Add the barrier if requested.
2246 if (NeedsBarrier)
2247 createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
2248 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
2249
2250#ifndef NDEBUG
2251 // Even though we currently do not support applying additional methods to it,
2252 // the chunk loop should remain a canonical loop.
2253 CLI->assertOK();
2254#endif
2255
2256 return {DispatchAfter, DispatchAfter->getFirstInsertionPt()};
2257}
2258
2259OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop(
2260 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2261 bool NeedsBarrier, llvm::omp::ScheduleKind SchedKind,
2262 llvm::Value *ChunkSize, bool HasSimdModifier, bool HasMonotonicModifier,
2263 bool HasNonmonotonicModifier, bool HasOrderedClause) {
2264 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
2265 SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
2266 HasNonmonotonicModifier, HasOrderedClause);
2267
2268 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
2269 OMPScheduleType::ModifierOrdered;
2270 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
2271 case OMPScheduleType::BaseStatic:
2272 assert(!ChunkSize && "No chunk size with static-chunked schedule")(static_cast <bool> (!ChunkSize && "No chunk size with static-chunked schedule"
) ? void (0) : __assert_fail ("!ChunkSize && \"No chunk size with static-chunked schedule\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2272, __extension__
__PRETTY_FUNCTION__))
;
2273 if (IsOrdered)
2274 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2275 NeedsBarrier, ChunkSize);
2276 // FIXME: Monotonicity ignored?
2277 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier);
2278
2279 case OMPScheduleType::BaseStaticChunked:
2280 if (IsOrdered)
2281 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2282 NeedsBarrier, ChunkSize);
2283 // FIXME: Monotonicity ignored?
2284 return applyStaticChunkedWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier,
2285 ChunkSize);
2286
2287 case OMPScheduleType::BaseRuntime:
2288 case OMPScheduleType::BaseAuto:
2289 case OMPScheduleType::BaseGreedy:
2290 case OMPScheduleType::BaseBalanced:
2291 case OMPScheduleType::BaseSteal:
2292 case OMPScheduleType::BaseGuidedSimd:
2293 case OMPScheduleType::BaseRuntimeSimd:
2294 assert(!ChunkSize &&(static_cast <bool> (!ChunkSize && "schedule type does not support user-defined chunk sizes"
) ? void (0) : __assert_fail ("!ChunkSize && \"schedule type does not support user-defined chunk sizes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2295, __extension__
__PRETTY_FUNCTION__))
2295 "schedule type does not support user-defined chunk sizes")(static_cast <bool> (!ChunkSize && "schedule type does not support user-defined chunk sizes"
) ? void (0) : __assert_fail ("!ChunkSize && \"schedule type does not support user-defined chunk sizes\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2295, __extension__
__PRETTY_FUNCTION__))
;
2296 [[fallthrough]];
2297 case OMPScheduleType::BaseDynamicChunked:
2298 case OMPScheduleType::BaseGuidedChunked:
2299 case OMPScheduleType::BaseGuidedIterativeChunked:
2300 case OMPScheduleType::BaseGuidedAnalyticalChunked:
2301 case OMPScheduleType::BaseStaticBalancedChunked:
2302 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
2303 NeedsBarrier, ChunkSize);
2304
2305 default:
2306 llvm_unreachable("Unknown/unimplemented schedule kind")::llvm::llvm_unreachable_internal("Unknown/unimplemented schedule kind"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2306)
;
2307 }
2308}
2309
2310/// Returns an LLVM function to call for initializing loop bounds using OpenMP
2311/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2312/// the runtime. Always interpret integers as unsigned similarly to
2313/// CanonicalLoopInfo.
2314static FunctionCallee
2315getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2316 unsigned Bitwidth = Ty->getIntegerBitWidth();
2317 if (Bitwidth == 32)
2318 return OMPBuilder.getOrCreateRuntimeFunction(
2319 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
2320 if (Bitwidth == 64)
2321 return OMPBuilder.getOrCreateRuntimeFunction(
2322 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
2323 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2323)
;
2324}
2325
2326/// Returns an LLVM function to call for updating the next loop using OpenMP
2327/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2328/// the runtime. Always interpret integers as unsigned similarly to
2329/// CanonicalLoopInfo.
2330static FunctionCallee
2331getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2332 unsigned Bitwidth = Ty->getIntegerBitWidth();
2333 if (Bitwidth == 32)
2334 return OMPBuilder.getOrCreateRuntimeFunction(
2335 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
2336 if (Bitwidth == 64)
2337 return OMPBuilder.getOrCreateRuntimeFunction(
2338 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
2339 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2339)
;
2340}
2341
2342/// Returns an LLVM function to call for finalizing the dynamic loop using
2343/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
2344/// interpret integers as unsigned similarly to CanonicalLoopInfo.
2345static FunctionCallee
2346getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2347 unsigned Bitwidth = Ty->getIntegerBitWidth();
2348 if (Bitwidth == 32)
2349 return OMPBuilder.getOrCreateRuntimeFunction(
2350 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
2351 if (Bitwidth == 64)
2352 return OMPBuilder.getOrCreateRuntimeFunction(
2353 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
2354 llvm_unreachable("unknown OpenMP loop iterator bitwidth")::llvm::llvm_unreachable_internal("unknown OpenMP loop iterator bitwidth"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2354)
;
2355}
2356
2357OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
2358 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2359 OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
2360 assert(CLI->isValid() && "Requires a valid canonical loop")(static_cast <bool> (CLI->isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("CLI->isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2360, __extension__
__PRETTY_FUNCTION__))
;
2361 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2362, __extension__
__PRETTY_FUNCTION__))
2362 "Require dedicated allocate IP")(static_cast <bool> (!isConflictIP(AllocaIP, CLI->getPreheaderIP
()) && "Require dedicated allocate IP") ? void (0) : __assert_fail
("!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && \"Require dedicated allocate IP\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2362, __extension__
__PRETTY_FUNCTION__))
;
2363 assert(isValidWorkshareLoopScheduleType(SchedType) &&(static_cast <bool> (isValidWorkshareLoopScheduleType(SchedType
) && "Require valid schedule type") ? void (0) : __assert_fail
("isValidWorkshareLoopScheduleType(SchedType) && \"Require valid schedule type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2364, __extension__
__PRETTY_FUNCTION__))
2364 "Require valid schedule type")(static_cast <bool> (isValidWorkshareLoopScheduleType(SchedType
) && "Require valid schedule type") ? void (0) : __assert_fail
("isValidWorkshareLoopScheduleType(SchedType) && \"Require valid schedule type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2364, __extension__
__PRETTY_FUNCTION__))
;
2365
2366 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
2367 OMPScheduleType::ModifierOrdered;
2368
2369 // Set up the source location value for OpenMP runtime.
2370 Builder.SetCurrentDebugLocation(DL);
2371
2372 uint32_t SrcLocStrSize;
2373 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2374 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2375
2376 // Declare useful OpenMP runtime functions.
2377 Value *IV = CLI->getIndVar();
2378 Type *IVTy = IV->getType();
2379 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(IVTy, M, *this);
2380 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(IVTy, M, *this);
2381
2382 // Allocate space for computed loop bounds as expected by the "init" function.
2383 Builder.restoreIP(AllocaIP);
2384 Type *I32Type = Type::getInt32Ty(M.getContext());
2385 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
2386 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
2387 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
2388 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
2389
2390 // At the end of the preheader, prepare for calling the "init" function by
2391 // storing the current loop bounds into the allocated space. A canonical loop
2392 // always iterates from 0 to trip-count with step 1. Note that "init" expects
2393 // and produces an inclusive upper bound.
2394 BasicBlock *PreHeader = CLI->getPreheader();
2395 Builder.SetInsertPoint(PreHeader->getTerminator());
2396 Constant *One = ConstantInt::get(IVTy, 1);
2397 Builder.CreateStore(One, PLowerBound);
2398 Value *UpperBound = CLI->getTripCount();
2399 Builder.CreateStore(UpperBound, PUpperBound);
2400 Builder.CreateStore(One, PStride);
2401
2402 BasicBlock *Header = CLI->getHeader();
2403 BasicBlock *Exit = CLI->getExit();
2404 BasicBlock *Cond = CLI->getCond();
2405 BasicBlock *Latch = CLI->getLatch();
2406 InsertPointTy AfterIP = CLI->getAfterIP();
2407
2408 // The CLI will be "broken" in the code below, as the loop is no longer
2409 // a valid canonical loop.
2410
2411 if (!Chunk)
2412 Chunk = One;
2413
2414 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
2415
2416 Constant *SchedulingType =
2417 ConstantInt::get(I32Type, static_cast<int>(SchedType));
2418
2419 // Call the "init" function.
2420 Builder.CreateCall(DynamicInit,
2421 {SrcLoc, ThreadNum, SchedulingType, /* LowerBound */ One,
2422 UpperBound, /* step */ One, Chunk});
2423
2424 // An outer loop around the existing one.
2425 BasicBlock *OuterCond = BasicBlock::Create(
2426 PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond",
2427 PreHeader->getParent());
2428 // This needs to be 32-bit always, so can't use the IVTy Zero above.
2429 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());
2430 Value *Res =
2431 Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter,
2432 PLowerBound, PUpperBound, PStride});
2433 Constant *Zero32 = ConstantInt::get(I32Type, 0);
2434 Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);
2435 Value *LowerBound =
2436 Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");
2437 Builder.CreateCondBr(MoreWork, Header, Exit);
2438
2439 // Change PHI-node in loop header to use outer cond rather than preheader,
2440 // and set IV to the LowerBound.
2441 Instruction *Phi = &Header->front();
2442 auto *PI = cast<PHINode>(Phi);
2443 PI->setIncomingBlock(0, OuterCond);
2444 PI->setIncomingValue(0, LowerBound);
2445
2446 // Then set the pre-header to jump to the OuterCond
2447 Instruction *Term = PreHeader->getTerminator();
2448 auto *Br = cast<BranchInst>(Term);
2449 Br->setSuccessor(0, OuterCond);
2450
2451 // Modify the inner condition:
2452 // * Use the UpperBound returned from the DynamicNext call.
2453 // * jump to the loop outer loop when done with one of the inner loops.
2454 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());
2455 UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");
2456 Instruction *Comp = &*Builder.GetInsertPoint();
2457 auto *CI = cast<CmpInst>(Comp);
2458 CI->setOperand(1, UpperBound);
2459 // Redirect the inner exit to branch to outer condition.
2460 Instruction *Branch = &Cond->back();
2461 auto *BI = cast<BranchInst>(Branch);
2462 assert(BI->getSuccessor(1) == Exit)(static_cast <bool> (BI->getSuccessor(1) == Exit) ? void
(0) : __assert_fail ("BI->getSuccessor(1) == Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 2462, __extension__ __PRETTY_FUNCTION__))
;
2463 BI->setSuccessor(1, OuterCond);
2464
2465 // Call the "fini" function if "ordered" is present in wsloop directive.
2466 if (Ordered) {
2467 Builder.SetInsertPoint(&Latch->back());
2468 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(IVTy, M, *this);
2469 Builder.CreateCall(DynamicFini, {SrcLoc, ThreadNum});
2470 }
2471
2472 // Add the barrier if requested.
2473 if (NeedsBarrier) {
2474 Builder.SetInsertPoint(&Exit->back());
2475 createBarrier(LocationDescription(Builder.saveIP(), DL),
2476 omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
2477 /* CheckCancelFlag */ false);
2478 }
2479
2480 CLI->invalidate();
2481 return AfterIP;
2482}
2483
2484/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
2485/// after this \p OldTarget will be orphaned.
2486static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
2487 BasicBlock *NewTarget, DebugLoc DL) {
2488 for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
2489 redirectTo(Pred, NewTarget, DL);
2490}
2491
2492/// Determine which blocks in \p BBs are reachable from outside and remove the
2493/// ones that are not reachable from the function.
2494static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
2495 SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
2496 auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
2497 for (Use &U : BB->uses()) {
2498 auto *UseInst = dyn_cast<Instruction>(U.getUser());
2499 if (!UseInst)
2500 continue;
2501 if (BBsToErase.count(UseInst->getParent()))
2502 continue;
2503 return true;
2504 }
2505 return false;
2506 };
2507
2508 while (true) {
2509 bool Changed = false;
2510 for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
2511 if (HasRemainingUses(BB)) {
2512 BBsToErase.erase(BB);
2513 Changed = true;
2514 }
2515 }
2516 if (!Changed)
2517 break;
2518 }
2519
2520 SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
2521 DeleteDeadBlocks(BBVec);
2522}
2523
2524CanonicalLoopInfo *
2525OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
2526 InsertPointTy ComputeIP) {
2527 assert(Loops.size() >= 1 && "At least one loop required")(static_cast <bool> (Loops.size() >= 1 && "At least one loop required"
) ? void (0) : __assert_fail ("Loops.size() >= 1 && \"At least one loop required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2527, __extension__
__PRETTY_FUNCTION__))
;
2528 size_t NumLoops = Loops.size();
2529
2530 // Nothing to do if there is already just one loop.
2531 if (NumLoops == 1)
2532 return Loops.front();
2533
2534 CanonicalLoopInfo *Outermost = Loops.front();
2535 CanonicalLoopInfo *Innermost = Loops.back();
2536 BasicBlock *OrigPreheader = Outermost->getPreheader();
2537 BasicBlock *OrigAfter = Outermost->getAfter();
2538 Function *F = OrigPreheader->getParent();
2539
2540 // Loop control blocks that may become orphaned later.
2541 SmallVector<BasicBlock *, 12> OldControlBBs;
2542 OldControlBBs.reserve(6 * Loops.size());
2543 for (CanonicalLoopInfo *Loop : Loops)
2544 Loop->collectControlBlocks(OldControlBBs);
2545
2546 // Setup the IRBuilder for inserting the trip count computation.
2547 Builder.SetCurrentDebugLocation(DL);
2548 if (ComputeIP.isSet())
2549 Builder.restoreIP(ComputeIP);
2550 else
2551 Builder.restoreIP(Outermost->getPreheaderIP());
2552
2553 // Derive the collapsed' loop trip count.
2554 // TODO: Find common/largest indvar type.
2555 Value *CollapsedTripCount = nullptr;
2556 for (CanonicalLoopInfo *L : Loops) {
2557 assert(L->isValid() &&(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2558, __extension__
__PRETTY_FUNCTION__))
2558 "All loops to collapse must be valid canonical loops")(static_cast <bool> (L->isValid() && "All loops to collapse must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All loops to collapse must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2558, __extension__
__PRETTY_FUNCTION__))
;
2559 Value *OrigTripCount = L->getTripCount();
2560 if (!CollapsedTripCount) {
2561 CollapsedTripCount = OrigTripCount;
2562 continue;
2563 }
2564
2565 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
2566 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
2567 {}, /*HasNUW=*/true);
2568 }
2569
2570 // Create the collapsed loop control flow.
2571 CanonicalLoopInfo *Result =
2572 createLoopSkeleton(DL, CollapsedTripCount, F,
2573 OrigPreheader->getNextNode(), OrigAfter, "collapsed");
2574
2575 // Build the collapsed loop body code.
2576 // Start with deriving the input loop induction variables from the collapsed
2577 // one, using a divmod scheme. To preserve the original loops' order, the
2578 // innermost loop use the least significant bits.
2579 Builder.restoreIP(Result->getBodyIP());
2580
2581 Value *Leftover = Result->getIndVar();
2582 SmallVector<Value *> NewIndVars;
2583 NewIndVars.resize(NumLoops);
2584 for (int i = NumLoops - 1; i >= 1; --i) {
2585 Value *OrigTripCount = Loops[i]->getTripCount();
2586
2587 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
2588 NewIndVars[i] = NewIndVar;
2589
2590 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
2591 }
2592 // Outermost loop gets all the remaining bits.
2593 NewIndVars[0] = Leftover;
2594
2595 // Construct the loop body control flow.
2596 // We progressively construct the branch structure following in direction of
2597 // the control flow, from the leading in-between code, the loop nest body, the
2598 // trailing in-between code, and rejoining the collapsed loop's latch.
2599 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
2600 // the ContinueBlock is set, continue with that block. If ContinuePred, use
2601 // its predecessors as sources.
2602 BasicBlock *ContinueBlock = Result->getBody();
2603 BasicBlock *ContinuePred = nullptr;
2604 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
2605 BasicBlock *NextSrc) {
2606 if (ContinueBlock)
2607 redirectTo(ContinueBlock, Dest, DL);
2608 else
2609 redirectAllPredecessorsTo(ContinuePred, Dest, DL);
2610
2611 ContinueBlock = nullptr;
2612 ContinuePred = NextSrc;
2613 };
2614
2615 // The code before the nested loop of each level.
2616 // Because we are sinking it into the nest, it will be executed more often
2617 // that the original loop. More sophisticated schemes could keep track of what
2618 // the in-between code is and instantiate it only once per thread.
2619 for (size_t i = 0; i < NumLoops - 1; ++i)
2620 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
2621
2622 // Connect the loop nest body.
2623 ContinueWith(Innermost->getBody(), Innermost->getLatch());
2624
2625 // The code after the nested loop at each level.
2626 for (size_t i = NumLoops - 1; i > 0; --i)
2627 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
2628
2629 // Connect the finished loop to the collapsed loop latch.
2630 ContinueWith(Result->getLatch(), nullptr);
2631
2632 // Replace the input loops with the new collapsed loop.
2633 redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
2634 redirectTo(Result->getAfter(), Outermost->getAfter(), DL);
2635
2636 // Replace the input loop indvars with the derived ones.
2637 for (size_t i = 0; i < NumLoops; ++i)
2638 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
2639
2640 // Remove unused parts of the input loops.
2641 removeUnusedBlocksFromParent(OldControlBBs);
2642
2643 for (CanonicalLoopInfo *L : Loops)
2644 L->invalidate();
2645
2646#ifndef NDEBUG
2647 Result->assertOK();
2648#endif
2649 return Result;
2650}
2651
2652std::vector<CanonicalLoopInfo *>
2653OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
2654 ArrayRef<Value *> TileSizes) {
2655 assert(TileSizes.size() == Loops.size() &&(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2656, __extension__
__PRETTY_FUNCTION__))
2656 "Must pass as many tile sizes as there are loops")(static_cast <bool> (TileSizes.size() == Loops.size() &&
"Must pass as many tile sizes as there are loops") ? void (0
) : __assert_fail ("TileSizes.size() == Loops.size() && \"Must pass as many tile sizes as there are loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2656, __extension__
__PRETTY_FUNCTION__))
;
2657 int NumLoops = Loops.size();
2658 assert(NumLoops >= 1 && "At least one loop to tile required")(static_cast <bool> (NumLoops >= 1 && "At least one loop to tile required"
) ? void (0) : __assert_fail ("NumLoops >= 1 && \"At least one loop to tile required\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2658, __extension__
__PRETTY_FUNCTION__))
;
2659
2660 CanonicalLoopInfo *OutermostLoop = Loops.front();
2661 CanonicalLoopInfo *InnermostLoop = Loops.back();
2662 Function *F = OutermostLoop->getBody()->getParent();
2663 BasicBlock *InnerEnter = InnermostLoop->getBody();
2664 BasicBlock *InnerLatch = InnermostLoop->getLatch();
2665
2666 // Loop control blocks that may become orphaned later.
2667 SmallVector<BasicBlock *, 12> OldControlBBs;
2668 OldControlBBs.reserve(6 * Loops.size());
2669 for (CanonicalLoopInfo *Loop : Loops)
2670 Loop->collectControlBlocks(OldControlBBs);
2671
2672 // Collect original trip counts and induction variable to be accessible by
2673 // index. Also, the structure of the original loops is not preserved during
2674 // the construction of the tiled loops, so do it before we scavenge the BBs of
2675 // any original CanonicalLoopInfo.
2676 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
2677 for (CanonicalLoopInfo *L : Loops) {
2678 assert(L->isValid() && "All input loops must be valid canonical loops")(static_cast <bool> (L->isValid() && "All input loops must be valid canonical loops"
) ? void (0) : __assert_fail ("L->isValid() && \"All input loops must be valid canonical loops\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2678, __extension__
__PRETTY_FUNCTION__))
;
2679 OrigTripCounts.push_back(L->getTripCount());
2680 OrigIndVars.push_back(L->getIndVar());
2681 }
2682
2683 // Collect the code between loop headers. These may contain SSA definitions
2684 // that are used in the loop nest body. To be usable with in the innermost
2685 // body, these BasicBlocks will be sunk into the loop nest body. That is,
2686 // these instructions may be executed more often than before the tiling.
2687 // TODO: It would be sufficient to only sink them into body of the
2688 // corresponding tile loop.
2689 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
2690 for (int i = 0; i < NumLoops - 1; ++i) {
2691 CanonicalLoopInfo *Surrounding = Loops[i];
2692 CanonicalLoopInfo *Nested = Loops[i + 1];
2693
2694 BasicBlock *EnterBB = Surrounding->getBody();
2695 BasicBlock *ExitBB = Nested->getHeader();
2696 InbetweenCode.emplace_back(EnterBB, ExitBB);
2697 }
2698
2699 // Compute the trip counts of the floor loops.
2700 Builder.SetCurrentDebugLocation(DL);
2701 Builder.restoreIP(OutermostLoop->getPreheaderIP());
2702 SmallVector<Value *, 4> FloorCount, FloorRems;
2703 for (int i = 0; i < NumLoops; ++i) {
2704 Value *TileSize = TileSizes[i];
2705 Value *OrigTripCount = OrigTripCounts[i];
2706 Type *IVType = OrigTripCount->getType();
2707
2708 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
2709 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
2710
2711 // 0 if tripcount divides the tilesize, 1 otherwise.
2712 // 1 means we need an additional iteration for a partial tile.
2713 //
2714 // Unfortunately we cannot just use the roundup-formula
2715 // (tripcount + tilesize - 1)/tilesize
2716 // because the summation might overflow. We do not want introduce undefined
2717 // behavior when the untiled loop nest did not.
2718 Value *FloorTripOverflow =
2719 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
2720
2721 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
2722 FloorTripCount =
2723 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
2724 "omp_floor" + Twine(i) + ".tripcount", true);
2725
2726 // Remember some values for later use.
2727 FloorCount.push_back(FloorTripCount);
2728 FloorRems.push_back(FloorTripRem);
2729 }
2730
2731 // Generate the new loop nest, from the outermost to the innermost.
2732 std::vector<CanonicalLoopInfo *> Result;
2733 Result.reserve(NumLoops * 2);
2734
2735 // The basic block of the surrounding loop that enters the nest generated
2736 // loop.
2737 BasicBlock *Enter = OutermostLoop->getPreheader();
2738
2739 // The basic block of the surrounding loop where the inner code should
2740 // continue.
2741 BasicBlock *Continue = OutermostLoop->getAfter();
2742
2743 // Where the next loop basic block should be inserted.
2744 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
2745
2746 auto EmbeddNewLoop =
2747 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
2748 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
2749 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
2750 DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
2751 redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
2752 redirectTo(EmbeddedLoop->getAfter(), Continue, DL);
2753
2754 // Setup the position where the next embedded loop connects to this loop.
2755 Enter = EmbeddedLoop->getBody();
2756 Continue = EmbeddedLoop->getLatch();
2757 OutroInsertBefore = EmbeddedLoop->getLatch();
2758 return EmbeddedLoop;
2759 };
2760
2761 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
2762 const Twine &NameBase) {
2763 for (auto P : enumerate(TripCounts)) {
2764 CanonicalLoopInfo *EmbeddedLoop =
2765 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
2766 Result.push_back(EmbeddedLoop);
2767 }
2768 };
2769
2770 EmbeddNewLoops(FloorCount, "floor");
2771
2772 // Within the innermost floor loop, emit the code that computes the tile
2773 // sizes.
2774 Builder.SetInsertPoint(Enter->getTerminator());
2775 SmallVector<Value *, 4> TileCounts;
2776 for (int i = 0; i < NumLoops; ++i) {
2777 CanonicalLoopInfo *FloorLoop = Result[i];
2778 Value *TileSize = TileSizes[i];
2779
2780 Value *FloorIsEpilogue =
2781 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
2782 Value *TileTripCount =
2783 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
2784
2785 TileCounts.push_back(TileTripCount);
2786 }
2787
2788 // Create the tile loops.
2789 EmbeddNewLoops(TileCounts, "tile");
2790
2791 // Insert the inbetween code into the body.
2792 BasicBlock *BodyEnter = Enter;
2793 BasicBlock *BodyEntered = nullptr;
2794 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
2795 BasicBlock *EnterBB = P.first;
2796 BasicBlock *ExitBB = P.second;
2797
2798 if (BodyEnter)
2799 redirectTo(BodyEnter, EnterBB, DL);
2800 else
2801 redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);
2802
2803 BodyEnter = nullptr;
2804 BodyEntered = ExitBB;
2805 }
2806
2807 // Append the original loop nest body into the generated loop nest body.
2808 if (BodyEnter)
2809 redirectTo(BodyEnter, InnerEnter, DL);
2810 else
2811 redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
2812 redirectAllPredecessorsTo(InnerLatch, Continue, DL);
2813
2814 // Replace the original induction variable with an induction variable computed
2815 // from the tile and floor induction variables.
2816 Builder.restoreIP(Result.back()->getBodyIP());
2817 for (int i = 0; i < NumLoops; ++i) {
2818 CanonicalLoopInfo *FloorLoop = Result[i];
2819 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
2820 Value *OrigIndVar = OrigIndVars[i];
2821 Value *Size = TileSizes[i];
2822
2823 Value *Scale =
2824 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
2825 Value *Shift =
2826 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
2827 OrigIndVar->replaceAllUsesWith(Shift);
2828 }
2829
2830 // Remove unused parts of the original loops.
2831 removeUnusedBlocksFromParent(OldControlBBs);
2832
2833 for (CanonicalLoopInfo *L : Loops)
2834 L->invalidate();
2835
2836#ifndef NDEBUG
2837 for (CanonicalLoopInfo *GenL : Result)
2838 GenL->assertOK();
2839#endif
2840 return Result;
2841}
2842
2843/// Attach metadata \p Properties to the basic block described by \p BB. If the
2844/// basic block already has metadata, the basic block properties are appended.
2845static void addBasicBlockMetadata(BasicBlock *BB,
2846 ArrayRef<Metadata *> Properties) {
2847 // Nothing to do if no property to attach.
2848 if (Properties.empty())
2849 return;
2850
2851 LLVMContext &Ctx = BB->getContext();
2852 SmallVector<Metadata *> NewProperties;
2853 NewProperties.push_back(nullptr);
2854
2855 // If the basic block already has metadata, prepend it to the new metadata.
2856 MDNode *Existing = BB->getTerminator()->getMetadata(LLVMContext::MD_loop);
2857 if (Existing)
2858 append_range(NewProperties, drop_begin(Existing->operands(), 1));
2859
2860 append_range(NewProperties, Properties);
2861 MDNode *BasicBlockID = MDNode::getDistinct(Ctx, NewProperties);
2862 BasicBlockID->replaceOperandWith(0, BasicBlockID);
2863
2864 BB->getTerminator()->setMetadata(LLVMContext::MD_loop, BasicBlockID);
2865}
2866
2867/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
2868/// loop already has metadata, the loop properties are appended.
2869static void addLoopMetadata(CanonicalLoopInfo *Loop,
2870 ArrayRef<Metadata *> Properties) {
2871 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo")(static_cast <bool> (Loop->isValid() && "Expecting a valid CanonicalLoopInfo"
) ? void (0) : __assert_fail ("Loop->isValid() && \"Expecting a valid CanonicalLoopInfo\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2871, __extension__
__PRETTY_FUNCTION__))
;
2872
2873 // Attach metadata to the loop's latch
2874 BasicBlock *Latch = Loop->getLatch();
2875 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch")(static_cast <bool> (Latch && "A valid CanonicalLoopInfo must have a unique latch"
) ? void (0) : __assert_fail ("Latch && \"A valid CanonicalLoopInfo must have a unique latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2875, __extension__
__PRETTY_FUNCTION__))
;
2876 addBasicBlockMetadata(Latch, Properties);
2877}
2878
2879/// Attach llvm.access.group metadata to the memref instructions of \p Block
2880static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup,
2881 LoopInfo &LI) {
2882 for (Instruction &I : *Block) {
2883 if (I.mayReadOrWriteMemory()) {
2884 // TODO: This instruction may already have access group from
2885 // other pragmas e.g. #pragma clang loop vectorize. Append
2886 // so that the existing metadata is not overwritten.
2887 I.setMetadata(LLVMContext::MD_access_group, AccessGroup);
2888 }
2889 }
2890}
2891
2892void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
2893 LLVMContext &Ctx = Builder.getContext();
2894 addLoopMetadata(
2895 Loop, {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2896 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))});
2897}
2898
2899void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
2900 LLVMContext &Ctx = Builder.getContext();
2901 addLoopMetadata(
2902 Loop, {
2903 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
2904 });
2905}
2906
2907void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
2908 Value *IfCond, ValueToValueMapTy &VMap,
2909 const Twine &NamePrefix) {
2910 Function *F = CanonicalLoop->getFunction();
2911
2912 // Define where if branch should be inserted
2913 Instruction *SplitBefore;
2914 if (Instruction::classof(IfCond)) {
2915 SplitBefore = dyn_cast<Instruction>(IfCond);
2916 } else {
2917 SplitBefore = CanonicalLoop->getPreheader()->getTerminator();
2918 }
2919
2920 // TODO: We should not rely on pass manager. Currently we use pass manager
2921 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
2922 // object. We should have a method which returns all blocks between
2923 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
2924 FunctionAnalysisManager FAM;
2925 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
2926 FAM.registerPass([]() { return LoopAnalysis(); });
2927 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
2928
2929 // Get the loop which needs to be cloned
2930 LoopAnalysis LIA;
2931 LoopInfo &&LI = LIA.run(*F, FAM);
2932 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());
2933
2934 // Create additional blocks for the if statement
2935 BasicBlock *Head = SplitBefore->getParent();
2936 Instruction *HeadOldTerm = Head->getTerminator();
2937 llvm::LLVMContext &C = Head->getContext();
2938 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
2939 C, NamePrefix + ".if.then", Head->getParent(), Head->getNextNode());
2940 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
2941 C, NamePrefix + ".if.else", Head->getParent(), CanonicalLoop->getExit());
2942
2943 // Create if condition branch.
2944 Builder.SetInsertPoint(HeadOldTerm);
2945 Instruction *BrInstr =
2946 Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);
2947 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
2948 // Then block contains branch to omp loop which needs to be vectorized
2949 spliceBB(IP, ThenBlock, false);
2950 ThenBlock->replaceSuccessorsPhiUsesWith(Head, ThenBlock);
2951
2952 Builder.SetInsertPoint(ElseBlock);
2953
2954 // Clone loop for the else branch
2955 SmallVector<BasicBlock *, 8> NewBlocks;
2956
2957 VMap[CanonicalLoop->getPreheader()] = ElseBlock;
2958 for (BasicBlock *Block : L->getBlocks()) {
2959 BasicBlock *NewBB = CloneBasicBlock(Block, VMap, "", F);
2960 NewBB->moveBefore(CanonicalLoop->getExit());
2961 VMap[Block] = NewBB;
2962 NewBlocks.push_back(NewBB);
2963 }
2964 remapInstructionsInBlocks(NewBlocks, VMap);
2965 Builder.CreateBr(NewBlocks.front());
2966}
2967
2968void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, Value *IfCond,
2969 ConstantInt *Simdlen, ConstantInt *Safelen) {
2970 LLVMContext &Ctx = Builder.getContext();
2971
2972 Function *F = CanonicalLoop->getFunction();
2973
2974 // TODO: We should not rely on pass manager. Currently we use pass manager
2975 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
2976 // object. We should have a method which returns all blocks between
2977 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
2978 FunctionAnalysisManager FAM;
2979 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
2980 FAM.registerPass([]() { return LoopAnalysis(); });
2981 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
2982
2983 LoopAnalysis LIA;
2984 LoopInfo &&LI = LIA.run(*F, FAM);
2985
2986 Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());
2987
2988 if (IfCond) {
2989 ValueToValueMapTy VMap;
2990 createIfVersion(CanonicalLoop, IfCond, VMap, "simd");
2991 // Add metadata to the cloned loop which disables vectorization
2992 Value *MappedLatch = VMap.lookup(CanonicalLoop->getLatch());
2993 assert(MappedLatch &&(static_cast <bool> (MappedLatch && "Cannot find value which corresponds to original loop latch"
) ? void (0) : __assert_fail ("MappedLatch && \"Cannot find value which corresponds to original loop latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2994, __extension__
__PRETTY_FUNCTION__))
2994 "Cannot find value which corresponds to original loop latch")(static_cast <bool> (MappedLatch && "Cannot find value which corresponds to original loop latch"
) ? void (0) : __assert_fail ("MappedLatch && \"Cannot find value which corresponds to original loop latch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2994, __extension__
__PRETTY_FUNCTION__))
;
2995 assert(isa<BasicBlock>(MappedLatch) &&(static_cast <bool> (isa<BasicBlock>(MappedLatch)
&& "Cannot cast mapped latch block value to BasicBlock"
) ? void (0) : __assert_fail ("isa<BasicBlock>(MappedLatch) && \"Cannot cast mapped latch block value to BasicBlock\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2996, __extension__
__PRETTY_FUNCTION__))
2996 "Cannot cast mapped latch block value to BasicBlock")(static_cast <bool> (isa<BasicBlock>(MappedLatch)
&& "Cannot cast mapped latch block value to BasicBlock"
) ? void (0) : __assert_fail ("isa<BasicBlock>(MappedLatch) && \"Cannot cast mapped latch block value to BasicBlock\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 2996, __extension__
__PRETTY_FUNCTION__))
;
2997 BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(MappedLatch);
2998 ConstantAsMetadata *BoolConst =
2999 ConstantAsMetadata::get(ConstantInt::getFalse(Type::getInt1Ty(Ctx)));
3000 addBasicBlockMetadata(
3001 NewLatchBlock,
3002 {MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
3003 BoolConst})});
3004 }
3005
3006 SmallSet<BasicBlock *, 8> Reachable;
3007
3008 // Get the basic blocks from the loop in which memref instructions
3009 // can be found.
3010 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
3011 // preferably without running any passes.
3012 for (BasicBlock *Block : L->getBlocks()) {
3013 if (Block == CanonicalLoop->getCond() ||
3014 Block == CanonicalLoop->getHeader())
3015 continue;
3016 Reachable.insert(Block);
3017 }
3018
3019 SmallVector<Metadata *> LoopMDList;
3020
3021 // In presence of finite 'safelen', it may be unsafe to mark all
3022 // the memory instructions parallel, because loop-carried
3023 // dependences of 'safelen' iterations are possible.
3024 if (Safelen == nullptr) {
3025 // Add access group metadata to memory-access instructions.
3026 MDNode *AccessGroup = MDNode::getDistinct(Ctx, {});
3027 for (BasicBlock *BB : Reachable)
3028 addSimdMetadata(BB, AccessGroup, LI);
3029 // TODO: If the loop has existing parallel access metadata, have
3030 // to combine two lists.
3031 LoopMDList.push_back(MDNode::get(
3032 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccessGroup}));
3033 }
3034
3035 // Use the above access group metadata to create loop level
3036 // metadata, which should be distinct for each loop.
3037 ConstantAsMetadata *BoolConst =
3038 ConstantAsMetadata::get(ConstantInt::getTrue(Type::getInt1Ty(Ctx)));
3039 LoopMDList.push_back(MDNode::get(
3040 Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), BoolConst}));
3041
3042 if (Simdlen || Safelen) {
3043 // If both simdlen and safelen clauses are specified, the value of the
3044 // simdlen parameter must be less than or equal to the value of the safelen
3045 // parameter. Therefore, use safelen only in the absence of simdlen.
3046 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
3047 LoopMDList.push_back(
3048 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.width"),
3049 ConstantAsMetadata::get(VectorizeWidth)}));
3050 }
3051
3052 addLoopMetadata(CanonicalLoop, LoopMDList);
3053}
3054
3055/// Create the TargetMachine object to query the backend for optimization
3056/// preferences.
3057///
3058/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
3059/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
3060/// needed for the LLVM pass pipline. We use some default options to avoid
3061/// having to pass too many settings from the frontend that probably do not
3062/// matter.
3063///
3064/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
3065/// method. If we are going to use TargetMachine for more purposes, especially
3066/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
3067/// might become be worth requiring front-ends to pass on their TargetMachine,
3068/// or at least cache it between methods. Note that while fontends such as Clang
3069/// have just a single main TargetMachine per translation unit, "target-cpu" and
3070/// "target-features" that determine the TargetMachine are per-function and can
3071/// be overrided using __attribute__((target("OPTIONS"))).
3072static std::unique_ptr<TargetMachine>
3073createTargetMachine(Function *F, CodeGenOpt::Level OptLevel) {
3074 Module *M = F->getParent();
3075
3076 StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString();
3077 StringRef Features = F->getFnAttribute("target-features").getValueAsString();
3078 const std::string &Triple = M->getTargetTriple();
3079
3080 std::string Error;
3081 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
3082 if (!TheTarget)
3083 return {};
3084
3085 llvm::TargetOptions Options;
3086 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
3087 Triple, CPU, Features, Options, /*RelocModel=*/None, /*CodeModel=*/None,
3088 OptLevel));
3089}
3090
3091/// Heuristically determine the best-performant unroll factor for \p CLI. This
3092/// depends on the target processor. We are re-using the same heuristics as the
3093/// LoopUnrollPass.
3094static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
3095 Function *F = CLI->getFunction();
3096
3097 // Assume the user requests the most aggressive unrolling, even if the rest of
3098 // the code is optimized using a lower setting.
3099 CodeGenOpt::Level OptLevel = CodeGenOpt::Aggressive;
3100 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
3101
3102 FunctionAnalysisManager FAM;
3103 FAM.registerPass([]() { return TargetLibraryAnalysis(); });
3104 FAM.registerPass([]() { return AssumptionAnalysis(); });
3105 FAM.registerPass([]() { return DominatorTreeAnalysis(); });
3106 FAM.registerPass([]() { return LoopAnalysis(); });
3107 FAM.registerPass([]() { return ScalarEvolutionAnalysis(); });
3108 FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
3109 TargetIRAnalysis TIRA;
3110 if (TM)
3111 TIRA = TargetIRAnalysis(
3112 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
3113 FAM.registerPass([&]() { return TIRA; });
3114
3115 TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM);
3116 ScalarEvolutionAnalysis SEA;
3117 ScalarEvolution &&SE = SEA.run(*F, FAM);
3118 DominatorTreeAnalysis DTA;
3119 DominatorTree &&DT = DTA.run(*F, FAM);
3120 LoopAnalysis LIA;
3121 LoopInfo &&LI = LIA.run(*F, FAM);
3122 AssumptionAnalysis ACT;
3123 AssumptionCache &&AC = ACT.run(*F, FAM);
3124 OptimizationRemarkEmitter ORE{F};
3125
3126 Loop *L = LI.getLoopFor(CLI->getHeader());
3127 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop")(static_cast <bool> (L && "Expecting CanonicalLoopInfo to be recognized as a loop"
) ? void (0) : __assert_fail ("L && \"Expecting CanonicalLoopInfo to be recognized as a loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3127, __extension__
__PRETTY_FUNCTION__))
;
3128
3129 TargetTransformInfo::UnrollingPreferences UP =
3130 gatherUnrollingPreferences(L, SE, TTI,
3131 /*BlockFrequencyInfo=*/nullptr,
3132 /*ProfileSummaryInfo=*/nullptr, ORE, OptLevel,
3133 /*UserThreshold=*/None,
3134 /*UserCount=*/None,
3135 /*UserAllowPartial=*/true,
3136 /*UserAllowRuntime=*/true,
3137 /*UserUpperBound=*/None,
3138 /*UserFullUnrollMaxCount=*/None);
3139
3140 UP.Force = true;
3141
3142 // Account for additional optimizations taking place before the LoopUnrollPass
3143 // would unroll the loop.
3144 UP.Threshold *= UnrollThresholdFactor;
3145 UP.PartialThreshold *= UnrollThresholdFactor;
3146
3147 // Use normal unroll factors even if the rest of the code is optimized for
3148 // size.
3149 UP.OptSizeThreshold = UP.Threshold;
3150 UP.PartialOptSizeThreshold = UP.PartialThreshold;
3151
3152 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3153 << " Threshold=" << UP.Threshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3154 << " PartialThreshold=" << UP.PartialThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3155 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3156 << " PartialOptSizeThreshold="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
3157 << UP.PartialOptSizeThreshold << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Unroll heuristic thresholds:\n"
<< " Threshold=" << UP.Threshold << "\n" <<
" PartialThreshold=" << UP.PartialThreshold << "\n"
<< " OptSizeThreshold=" << UP.OptSizeThreshold <<
"\n" << " PartialOptSizeThreshold=" << UP.PartialOptSizeThreshold
<< "\n"; } } while (false)
;
3158
3159 // Disable peeling.
3160 TargetTransformInfo::PeelingPreferences PP =
3161 gatherPeelingPreferences(L, SE, TTI,
3162 /*UserAllowPeeling=*/false,
3163 /*UserAllowProfileBasedPeeling=*/false,
3164 /*UnrollingSpecficValues=*/false);
3165
3166 SmallPtrSet<const Value *, 32> EphValues;
3167 CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
3168
3169 // Assume that reads and writes to stack variables can be eliminated by
3170 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
3171 // size.
3172 for (BasicBlock *BB : L->blocks()) {
3173 for (Instruction &I : *BB) {
3174 Value *Ptr;
3175 if (auto *Load = dyn_cast<LoadInst>(&I)) {
3176 Ptr = Load->getPointerOperand();
3177 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
3178 Ptr = Store->getPointerOperand();
3179 } else
3180 continue;
3181
3182 Ptr = Ptr->stripPointerCasts();
3183
3184 if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) {
3185 if (Alloca->getParent() == &F->getEntryBlock())
3186 EphValues.insert(&I);
3187 }
3188 }
3189 }
3190
3191 unsigned NumInlineCandidates;
3192 bool NotDuplicatable;
3193 bool Convergent;
3194 InstructionCost LoopSizeIC =
3195 ApproximateLoopSize(L, NumInlineCandidates, NotDuplicatable, Convergent,
3196 TTI, EphValues, UP.BEInsns);
3197 LLVM_DEBUG(dbgs() << "Estimated loop size is " << LoopSizeIC << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Estimated loop size is "
<< LoopSizeIC << "\n"; } } while (false)
;
3198
3199 // Loop is not unrollable if the loop contains certain instructions.
3200 if (NotDuplicatable || Convergent || !LoopSizeIC.isValid()) {
3201 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Loop not considered unrollable\n"
; } } while (false)
;
3202 return 1;
3203 }
3204 unsigned LoopSize = *LoopSizeIC.getValue();
3205
3206 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
3207 // be able to use it.
3208 int TripCount = 0;
3209 int MaxTripCount = 0;
3210 bool MaxOrZero = false;
3211 unsigned TripMultiple = 0;
3212
3213 bool UseUpperBound = false;
3214 computeUnrollCount(L, TTI, DT, &LI, SE, EphValues, &ORE, TripCount,
3215 MaxTripCount, MaxOrZero, TripMultiple, LoopSize, UP, PP,
3216 UseUpperBound);
3217 unsigned Factor = UP.Count;
3218 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { dbgs() << "Suggesting unroll factor of "
<< Factor << "\n"; } } while (false)
;
3219
3220 // This function returns 1 to signal to not unroll a loop.
3221 if (Factor == 0)
3222 return 1;
3223 return Factor;
3224}
3225
3226void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
3227 int32_t Factor,
3228 CanonicalLoopInfo **UnrolledCLI) {
3229 assert(Factor >= 0 && "Unroll factor must not be negative")(static_cast <bool> (Factor >= 0 && "Unroll factor must not be negative"
) ? void (0) : __assert_fail ("Factor >= 0 && \"Unroll factor must not be negative\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3229, __extension__
__PRETTY_FUNCTION__))
;
3230
3231 Function *F = Loop->getFunction();
3232 LLVMContext &Ctx = F->getContext();
3233
3234 // If the unrolled loop is not used for another loop-associated directive, it
3235 // is sufficient to add metadata for the LoopUnrollPass.
3236 if (!UnrolledCLI) {
3237 SmallVector<Metadata *, 2> LoopMetadata;
3238 LoopMetadata.push_back(
3239 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")));
3240
3241 if (Factor >= 1) {
3242 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3243 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
3244 LoopMetadata.push_back(MDNode::get(
3245 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst}));
3246 }
3247
3248 addLoopMetadata(Loop, LoopMetadata);
3249 return;
3250 }
3251
3252 // Heuristically determine the unroll factor.
3253 if (Factor == 0)
3254 Factor = computeHeuristicUnrollFactor(Loop);
3255
3256 // No change required with unroll factor 1.
3257 if (Factor == 1) {
3258 *UnrolledCLI = Loop;
3259 return;
3260 }
3261
3262 assert(Factor >= 2 &&(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3263, __extension__
__PRETTY_FUNCTION__))
3263 "unrolling only makes sense with a factor of 2 or larger")(static_cast <bool> (Factor >= 2 && "unrolling only makes sense with a factor of 2 or larger"
) ? void (0) : __assert_fail ("Factor >= 2 && \"unrolling only makes sense with a factor of 2 or larger\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3263, __extension__
__PRETTY_FUNCTION__))
;
3264
3265 Type *IndVarTy = Loop->getIndVarType();
3266
3267 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
3268 // unroll the inner loop.
3269 Value *FactorVal =
3270 ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor,
3271 /*isSigned=*/false));
3272 std::vector<CanonicalLoopInfo *> LoopNest =
3273 tileLoops(DL, {Loop}, {FactorVal});
3274 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling")(static_cast <bool> (LoopNest.size() == 2 && "Expect 2 loops after tiling"
) ? void (0) : __assert_fail ("LoopNest.size() == 2 && \"Expect 2 loops after tiling\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3274, __extension__
__PRETTY_FUNCTION__))
;
3275 *UnrolledCLI = LoopNest[0];
3276 CanonicalLoopInfo *InnerLoop = LoopNest[1];
3277
3278 // LoopUnrollPass can only fully unroll loops with constant trip count.
3279 // Unroll by the unroll factor with a fallback epilog for the remainder
3280 // iterations if necessary.
3281 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3282 ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
3283 addLoopMetadata(
3284 InnerLoop,
3285 {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
3286 MDNode::get(
3287 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst})});
3288
3289#ifndef NDEBUG
3290 (*UnrolledCLI)->assertOK();
3291#endif
3292}
3293
3294OpenMPIRBuilder::InsertPointTy
3295OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
3296 llvm::Value *BufSize, llvm::Value *CpyBuf,
3297 llvm::Value *CpyFn, llvm::Value *DidIt) {
3298 if (!updateToLocation(Loc))
3299 return Loc.IP;
3300
3301 uint32_t SrcLocStrSize;
3302 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3303 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3304 Value *ThreadId = getOrCreateThreadID(Ident);
3305
3306 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
3307
3308 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
3309
3310 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
3311 Builder.CreateCall(Fn, Args);
3312
3313 return Builder.saveIP();
3314}
3315
3316OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
3317 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3318 FinalizeCallbackTy FiniCB, bool IsNowait, llvm::Value *DidIt) {
3319
3320 if (!updateToLocation(Loc))
3321 return Loc.IP;
3322
3323 // If needed (i.e. not null), initialize `DidIt` with 0
3324 if (DidIt) {
3325 Builder.CreateStore(Builder.getInt32(0), DidIt);
3326 }
3327
3328 Directive OMPD = Directive::OMPD_single;
3329 uint32_t SrcLocStrSize;
3330 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3331 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3332 Value *ThreadId = getOrCreateThreadID(Ident);
3333 Value *Args[] = {Ident, ThreadId};
3334
3335 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
3336 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3337
3338 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
3339 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3340
3341 // generates the following:
3342 // if (__kmpc_single()) {
3343 // .... single region ...
3344 // __kmpc_end_single
3345 // }
3346 // __kmpc_barrier
3347
3348 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3349 /*Conditional*/ true,
3350 /*hasFinalize*/ true);
3351 if (!IsNowait)
3352 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
3353 omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
3354 /* CheckCancelFlag */ false);
3355 return Builder.saveIP();
3356}
3357
3358OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
3359 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3360 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
3361
3362 if (!updateToLocation(Loc))
3363 return Loc.IP;
3364
3365 Directive OMPD = Directive::OMPD_critical;
3366 uint32_t SrcLocStrSize;
3367 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3368 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3369 Value *ThreadId = getOrCreateThreadID(Ident);
3370 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
3371 Value *Args[] = {Ident, ThreadId, LockVar};
3372
3373 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
3374 Function *RTFn = nullptr;
3375 if (HintInst) {
3376 // Add Hint to entry Args and create call
3377 EnterArgs.push_back(HintInst);
3378 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
3379 } else {
3380 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
3381 }
3382 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
3383
3384 Function *ExitRTLFn =
3385 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
3386 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3387
3388 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3389 /*Conditional*/ false, /*hasFinalize*/ true);
3390}
3391
3392OpenMPIRBuilder::InsertPointTy
3393OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
3394 InsertPointTy AllocaIP, unsigned NumLoops,
3395 ArrayRef<llvm::Value *> StoreValues,
3396 const Twine &Name, bool IsDependSource) {
3397 for (const llvm::Value *SV : StoreValues)
3398 assert(SV->getType()->isIntegerTy(64) &&(static_cast <bool> (SV->getType()->isIntegerTy(64
) && "OpenMP runtime requires depend vec with i64 type"
) ? void (0) : __assert_fail ("SV->getType()->isIntegerTy(64) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3399, __extension__
__PRETTY_FUNCTION__))
3399 "OpenMP runtime requires depend vec with i64 type")(static_cast <bool> (SV->getType()->isIntegerTy(64
) && "OpenMP runtime requires depend vec with i64 type"
) ? void (0) : __assert_fail ("SV->getType()->isIntegerTy(64) && \"OpenMP runtime requires depend vec with i64 type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3399, __extension__
__PRETTY_FUNCTION__))
;
3400
3401 if (!updateToLocation(Loc))
3402 return Loc.IP;
3403
3404 // Allocate space for vector and generate alloc instruction.
3405 auto *ArrI64Ty = ArrayType::get(Int64, NumLoops);
3406 Builder.restoreIP(AllocaIP);
3407 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);
3408 ArgsBase->setAlignment(Align(8));
3409 Builder.restoreIP(Loc.IP);
3410
3411 // Store the index value with offset in depend vector.
3412 for (unsigned I = 0; I < NumLoops; ++I) {
3413 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
3414 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});
3415 StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);
3416 STInst->setAlignment(Align(8));
3417 }
3418
3419 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
3420 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});
3421
3422 uint32_t SrcLocStrSize;
3423 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3424 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3425 Value *ThreadId = getOrCreateThreadID(Ident);
3426 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
3427
3428 Function *RTLFn = nullptr;
3429 if (IsDependSource)
3430 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_post);
3431 else
3432 RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_wait);
3433 Builder.CreateCall(RTLFn, Args);
3434
3435 return Builder.saveIP();
3436}
3437
3438OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
3439 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
3440 FinalizeCallbackTy FiniCB, bool IsThreads) {
3441 if (!updateToLocation(Loc))
3442 return Loc.IP;
3443
3444 Directive OMPD = Directive::OMPD_ordered;
3445 Instruction *EntryCall = nullptr;
3446 Instruction *ExitCall = nullptr;
3447
3448 if (IsThreads) {
3449 uint32_t SrcLocStrSize;
3450 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3451 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3452 Value *ThreadId = getOrCreateThreadID(Ident);
3453 Value *Args[] = {Ident, ThreadId};
3454
3455 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_ordered);
3456 EntryCall = Builder.CreateCall(EntryRTLFn, Args);
3457
3458 Function *ExitRTLFn =
3459 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_ordered);
3460 ExitCall = Builder.CreateCall(ExitRTLFn, Args);
3461 }
3462
3463 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3464 /*Conditional*/ false, /*hasFinalize*/ true);
3465}
3466
3467OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
3468 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
3469 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
3470 bool HasFinalize, bool IsCancellable) {
3471
3472 if (HasFinalize)
3473 FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});
3474
3475 // Create inlined region's entry and body blocks, in preparation
3476 // for conditional creation
3477 BasicBlock *EntryBB = Builder.GetInsertBlock();
3478 Instruction *SplitPos = EntryBB->getTerminator();
3479 if (!isa_and_nonnull<BranchInst>(SplitPos))
3480 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
3481 BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
3482 BasicBlock *FiniBB =
3483 EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");
3484
3485 Builder.SetInsertPoint(EntryBB->getTerminator());
3486 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
3487
3488 // generate body
3489 BodyGenCB(/* AllocaIP */ InsertPointTy(),
3490 /* CodeGenIP */ Builder.saveIP());
3491
3492 // emit exit call and do any needed finalization.
3493 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
3494 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3496, __extension__
__PRETTY_FUNCTION__))
3495 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3496, __extension__
__PRETTY_FUNCTION__))
3496 "Unexpected control flow graph state!!")(static_cast <bool> (FiniBB->getTerminator()->getNumSuccessors
() == 1 && FiniBB->getTerminator()->getSuccessor
(0) == ExitBB && "Unexpected control flow graph state!!"
) ? void (0) : __assert_fail ("FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && \"Unexpected control flow graph state!!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3496, __extension__
__PRETTY_FUNCTION__))
;
3497 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
3498 assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3499, __extension__
__PRETTY_FUNCTION__))
3499 "Unexpected Control Flow State!")(static_cast <bool> (FiniBB->getUniquePredecessor()->
getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"
) ? void (0) : __assert_fail ("FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && \"Unexpected Control Flow State!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3499, __extension__
__PRETTY_FUNCTION__))
;
3500 MergeBlockIntoPredecessor(FiniBB);
3501
3502 // If we are skipping the region of a non conditional, remove the exit
3503 // block, and clear the builder's insertion point.
3504 assert(SplitPos->getParent() == ExitBB &&(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3505, __extension__
__PRETTY_FUNCTION__))
3505 "Unexpected Insertion point location!")(static_cast <bool> (SplitPos->getParent() == ExitBB
&& "Unexpected Insertion point location!") ? void (0
) : __assert_fail ("SplitPos->getParent() == ExitBB && \"Unexpected Insertion point location!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3505, __extension__
__PRETTY_FUNCTION__))
;
3506 auto merged = MergeBlockIntoPredecessor(ExitBB);
3507 BasicBlock *ExitPredBB = SplitPos->getParent();
3508 auto InsertBB = merged ? ExitPredBB : ExitBB;
3509 if (!isa_and_nonnull<BranchInst>(SplitPos))
3510 SplitPos->eraseFromParent();
3511 Builder.SetInsertPoint(InsertBB);
3512
3513 return Builder.saveIP();
3514}
3515
3516OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
3517 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
3518 // if nothing to do, Return current insertion point.
3519 if (!Conditional || !EntryCall)
3520 return Builder.saveIP();
3521
3522 BasicBlock *EntryBB = Builder.GetInsertBlock();
3523 Value *CallBool = Builder.CreateIsNotNull(EntryCall);
3524 auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
3525 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
3526
3527 // Emit thenBB and set the Builder's insertion point there for
3528 // body generation next. Place the block after the current block.
3529 Function *CurFn = EntryBB->getParent();
3530 CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);
3531
3532 // Move Entry branch to end of ThenBB, and replace with conditional
3533 // branch (If-stmt)
3534 Instruction *EntryBBTI = EntryBB->getTerminator();
3535 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
3536 EntryBBTI->removeFromParent();
3537 Builder.SetInsertPoint(UI);
3538 Builder.Insert(EntryBBTI);
3539 UI->eraseFromParent();
3540 Builder.SetInsertPoint(ThenBB->getTerminator());
3541
3542 // return an insertion point to ExitBB.
3543 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
3544}
3545
3546OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
3547 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
3548 bool HasFinalize) {
3549
3550 Builder.restoreIP(FinIP);
3551
3552 // If there is finalization to do, emit it before the exit call
3553 if (HasFinalize) {
3554 assert(!FinalizationStack.empty() &&(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3555, __extension__
__PRETTY_FUNCTION__))
3555 "Unexpected finalization stack state!")(static_cast <bool> (!FinalizationStack.empty() &&
"Unexpected finalization stack state!") ? void (0) : __assert_fail
("!FinalizationStack.empty() && \"Unexpected finalization stack state!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3555, __extension__
__PRETTY_FUNCTION__))
;
3556
3557 FinalizationInfo Fi = FinalizationStack.pop_back_val();
3558 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!")(static_cast <bool> (Fi.DK == OMPD && "Unexpected Directive for Finalization call!"
) ? void (0) : __assert_fail ("Fi.DK == OMPD && \"Unexpected Directive for Finalization call!\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3558, __extension__
__PRETTY_FUNCTION__))
;
3559
3560 Fi.FiniCB(FinIP);
3561
3562 BasicBlock *FiniBB = FinIP.getBlock();
3563 Instruction *FiniBBTI = FiniBB->getTerminator();
3564
3565 // set Builder IP for call creation
3566 Builder.SetInsertPoint(FiniBBTI);
3567 }
3568
3569 if (!ExitCall)
3570 return Builder.saveIP();
3571
3572 // place the Exitcall as last instruction before Finalization block terminator
3573 ExitCall->removeFromParent();
3574 Builder.Insert(ExitCall);
3575
3576 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
3577 ExitCall->getIterator());
3578}
3579
3580OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
3581 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
3582 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
3583 if (!IP.isSet())
3584 return IP;
3585
3586 IRBuilder<>::InsertPointGuard IPG(Builder);
3587
3588 // creates the following CFG structure
3589 // OMP_Entry : (MasterAddr != PrivateAddr)?
3590 // F T
3591 // | \
3592 // | copin.not.master
3593 // | /
3594 // v /
3595 // copyin.not.master.end
3596 // |
3597 // v
3598 // OMP.Entry.Next
3599
3600 BasicBlock *OMP_Entry = IP.getBlock();
3601 Function *CurFn = OMP_Entry->getParent();
3602 BasicBlock *CopyBegin =
3603 BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
3604 BasicBlock *CopyEnd = nullptr;
3605
3606 // If entry block is terminated, split to preserve the branch to following
3607 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
3608 if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
3609 CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
3610 "copyin.not.master.end");
3611 OMP_Entry->getTerminator()->eraseFromParent();
3612 } else {
3613 CopyEnd =
3614 BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
3615 }
3616
3617 Builder.SetInsertPoint(OMP_Entry);
3618 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
3619 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
3620 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
3621 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
3622
3623 Builder.SetInsertPoint(CopyBegin);
3624 if (BranchtoEnd)
3625 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
3626
3627 return Builder.saveIP();
3628}
3629
3630CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
3631 Value *Size, Value *Allocator,
3632 std::string Name) {
3633 IRBuilder<>::InsertPointGuard IPG(Builder);
3634 Builder.restoreIP(Loc.IP);
3635
3636 uint32_t SrcLocStrSize;
3637 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3638 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3639 Value *ThreadId = getOrCreateThreadID(Ident);
3640 Value *Args[] = {ThreadId, Size, Allocator};
3641
3642 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
3643
3644 return Builder.CreateCall(Fn, Args, Name);
3645}
3646
3647CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
3648 Value *Addr, Value *Allocator,
3649 std::string Name) {
3650 IRBuilder<>::InsertPointGuard IPG(Builder);
3651 Builder.restoreIP(Loc.IP);
3652
3653 uint32_t SrcLocStrSize;
3654 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3655 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3656 Value *ThreadId = getOrCreateThreadID(Ident);
3657 Value *Args[] = {ThreadId, Addr, Allocator};
3658 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
3659 return Builder.CreateCall(Fn, Args, Name);
3660}
3661
3662CallInst *OpenMPIRBuilder::createOMPInteropInit(
3663 const LocationDescription &Loc, Value *InteropVar,
3664 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
3665 Value *DependenceAddress, bool HaveNowaitClause) {
3666 IRBuilder<>::InsertPointGuard IPG(Builder);
3667 Builder.restoreIP(Loc.IP);
3668
3669 uint32_t SrcLocStrSize;
3670 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3671 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3672 Value *ThreadId = getOrCreateThreadID(Ident);
3673 if (Device == nullptr)
3674 Device = ConstantInt::get(Int32, -1);
3675 Constant *InteropTypeVal = ConstantInt::get(Int64, (int)InteropType);
3676 if (NumDependences == nullptr) {
3677 NumDependences = ConstantInt::get(Int32, 0);
3678 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3679 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3680 }
3681 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3682 Value *Args[] = {
3683 Ident, ThreadId, InteropVar, InteropTypeVal,
3684 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
3685
3686 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_init);
3687
3688 return Builder.CreateCall(Fn, Args);
3689}
3690
3691CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
3692 const LocationDescription &Loc, Value *InteropVar, Value *Device,
3693 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
3694 IRBuilder<>::InsertPointGuard IPG(Builder);
3695 Builder.restoreIP(Loc.IP);
3696
3697 uint32_t SrcLocStrSize;
3698 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3699 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3700 Value *ThreadId = getOrCreateThreadID(Ident);
3701 if (Device == nullptr)
3702 Device = ConstantInt::get(Int32, -1);
3703 if (NumDependences == nullptr) {
3704 NumDependences = ConstantInt::get(Int32, 0);
3705 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3706 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3707 }
3708 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3709 Value *Args[] = {
3710 Ident, ThreadId, InteropVar, Device,
3711 NumDependences, DependenceAddress, HaveNowaitClauseVal};
3712
3713 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_destroy);
3714
3715 return Builder.CreateCall(Fn, Args);
3716}
3717
3718CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
3719 Value *InteropVar, Value *Device,
3720 Value *NumDependences,
3721 Value *DependenceAddress,
3722 bool HaveNowaitClause) {
3723 IRBuilder<>::InsertPointGuard IPG(Builder);
3724 Builder.restoreIP(Loc.IP);
3725 uint32_t SrcLocStrSize;
3726 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3727 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3728 Value *ThreadId = getOrCreateThreadID(Ident);
3729 if (Device == nullptr)
3730 Device = ConstantInt::get(Int32, -1);
3731 if (NumDependences == nullptr) {
3732 NumDependences = ConstantInt::get(Int32, 0);
3733 PointerType *PointerTypeVar = Type::getInt8PtrTy(M.getContext());
3734 DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
3735 }
3736 Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
3737 Value *Args[] = {
3738 Ident, ThreadId, InteropVar, Device,
3739 NumDependences, DependenceAddress, HaveNowaitClauseVal};
3740
3741 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_use);
3742
3743 return Builder.CreateCall(Fn, Args);
3744}
3745
3746CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
3747 const LocationDescription &Loc, llvm::Value *Pointer,
3748 llvm::ConstantInt *Size, const llvm::Twine &Name) {
3749 IRBuilder<>::InsertPointGuard IPG(Builder);
3750 Builder.restoreIP(Loc.IP);
3751
3752 uint32_t SrcLocStrSize;
3753 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3754 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3755 Value *ThreadId = getOrCreateThreadID(Ident);
3756 Constant *ThreadPrivateCache =
3757 getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
3758 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
3759
3760 Function *Fn =
3761 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
3762
3763 return Builder.CreateCall(Fn, Args);
3764}
3765
3766OpenMPIRBuilder::InsertPointTy
3767OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
3768 bool RequiresFullRuntime) {
3769 if (!updateToLocation(Loc))
3770 return Loc.IP;
3771
3772 uint32_t SrcLocStrSize;
3773 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3774 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3775 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
3776 IntegerType::getInt8Ty(Int8->getContext()),
3777 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
3778 ConstantInt *UseGenericStateMachine =
3779 ConstantInt::getBool(Int32->getContext(), !IsSPMD);
3780 ConstantInt *RequiresFullRuntimeVal =
3781 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
3782
3783 Function *Fn = getOrCreateRuntimeFunctionPtr(
3784 omp::RuntimeFunction::OMPRTL___kmpc_target_init);
3785
3786 CallInst *ThreadKind = Builder.CreateCall(
3787 Fn, {Ident, IsSPMDVal, UseGenericStateMachine, RequiresFullRuntimeVal});
3788
3789 Value *ExecUserCode = Builder.CreateICmpEQ(
3790 ThreadKind, ConstantInt::get(ThreadKind->getType(), -1),
3791 "exec_user_code");
3792
3793 // ThreadKind = __kmpc_target_init(...)
3794 // if (ThreadKind == -1)
3795 // user_code
3796 // else
3797 // return;
3798
3799 auto *UI = Builder.CreateUnreachable();
3800 BasicBlock *CheckBB = UI->getParent();
3801 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry");
3802
3803 BasicBlock *WorkerExitBB = BasicBlock::Create(
3804 CheckBB->getContext(), "worker.exit", CheckBB->getParent());
3805 Builder.SetInsertPoint(WorkerExitBB);
3806 Builder.CreateRetVoid();
3807
3808 auto *CheckBBTI = CheckBB->getTerminator();
3809 Builder.SetInsertPoint(CheckBBTI);
3810 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);
3811
3812 CheckBBTI->eraseFromParent();
3813 UI->eraseFromParent();
3814
3815 // Continue in the "user_code" block, see diagram above and in
3816 // openmp/libomptarget/deviceRTLs/common/include/target.h .
3817 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
3818}
3819
3820void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
3821 bool IsSPMD,
3822 bool RequiresFullRuntime) {
3823 if (!updateToLocation(Loc))
3824 return;
3825
3826 uint32_t SrcLocStrSize;
3827 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3828 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3829 ConstantInt *IsSPMDVal = ConstantInt::getSigned(
3830 IntegerType::getInt8Ty(Int8->getContext()),
3831 IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
3832 ConstantInt *RequiresFullRuntimeVal =
3833 ConstantInt::getBool(Int32->getContext(), RequiresFullRuntime);
3834
3835 Function *Fn = getOrCreateRuntimeFunctionPtr(
3836 omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
3837
3838 Builder.CreateCall(Fn, {Ident, IsSPMDVal, RequiresFullRuntimeVal});
3839}
3840
3841std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
3842 StringRef FirstSeparator,
3843 StringRef Separator) {
3844 SmallString<128> Buffer;
3845 llvm::raw_svector_ostream OS(Buffer);
3846 StringRef Sep = FirstSeparator;
3847 for (StringRef Part : Parts) {
3848 OS << Sep << Part;
3849 Sep = Separator;
3850 }
3851 return OS.str().str();
3852}
3853
3854Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
3855 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3856 // TODO: Replace the twine arg with stringref to get rid of the conversion
3857 // logic. However This is taken from current implementation in clang as is.
3858 // Since this method is used in many places exclusively for OMP internal use
3859 // we will keep it as is for temporarily until we move all users to the
3860 // builder and then, if possible, fix it everywhere in one go.
3861 SmallString<256> Buffer;
3862 llvm::raw_svector_ostream Out(Buffer);
3863 Out << Name;
3864 StringRef RuntimeName = Out.str();
3865 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3866 if (Elem.second) {
3867 assert(cast<PointerType>(Elem.second->getType())(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3869, __extension__
__PRETTY_FUNCTION__))
3868 ->isOpaqueOrPointeeTypeMatches(Ty) &&(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3869, __extension__
__PRETTY_FUNCTION__))
3869 "OMP internal variable has different type than requested")(static_cast <bool> (cast<PointerType>(Elem.second
->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) &&
"OMP internal variable has different type than requested") ?
void (0) : __assert_fail ("cast<PointerType>(Elem.second->getType()) ->isOpaqueOrPointeeTypeMatches(Ty) && \"OMP internal variable has different type than requested\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3869, __extension__
__PRETTY_FUNCTION__))
;
3870 } else {
3871 // TODO: investigate the appropriate linkage type used for the global
3872 // variable for possibly changing that to internal or private, or maybe
3873 // create different versions of the function for different OMP internal
3874 // variables.
3875 Elem.second = new llvm::GlobalVariable(
3876 M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
3877 llvm::Constant::getNullValue(Ty), Elem.first(),
3878 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
3879 AddressSpace);
3880 }
3881
3882 return Elem.second;
3883}
3884
3885Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
3886 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3887 std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
3888 return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
3889}
3890
3891GlobalVariable *
3892OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
3893 std::string VarName) {
3894 llvm::Constant *MaptypesArrayInit =
3895 llvm::ConstantDataArray::get(M.getContext(), Mappings);
3896 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
3897 M, MaptypesArrayInit->getType(),
3898 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
3899 VarName);
3900 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3901 return MaptypesArrayGlobal;
3902}
3903
3904void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
3905 InsertPointTy AllocaIP,
3906 unsigned NumOperands,
3907 struct MapperAllocas &MapperAllocas) {
3908 if (!updateToLocation(Loc))
3909 return;
3910
3911 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
3912 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
3913 Builder.restoreIP(AllocaIP);
3914 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI8PtrTy);
3915 AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy);
3916 AllocaInst *ArgSizes = Builder.CreateAlloca(ArrI64Ty);
3917 Builder.restoreIP(Loc.IP);
3918 MapperAllocas.ArgsBase = ArgsBase;
3919 MapperAllocas.Args = Args;
3920 MapperAllocas.ArgSizes = ArgSizes;
3921}
3922
3923void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
3924 Function *MapperFunc, Value *SrcLocInfo,
3925 Value *MaptypesArg, Value *MapnamesArg,
3926 struct MapperAllocas &MapperAllocas,
3927 int64_t DeviceID, unsigned NumOperands) {
3928 if (!updateToLocation(Loc))
3929 return;
3930
3931 auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
3932 auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
3933 Value *ArgsBaseGEP =
3934 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
3935 {Builder.getInt32(0), Builder.getInt32(0)});
3936 Value *ArgsGEP =
3937 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
3938 {Builder.getInt32(0), Builder.getInt32(0)});
3939 Value *ArgSizesGEP =
3940 Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
3941 {Builder.getInt32(0), Builder.getInt32(0)});
3942 Value *NullPtr = Constant::getNullValue(Int8Ptr->getPointerTo());
3943 Builder.CreateCall(MapperFunc,
3944 {SrcLocInfo, Builder.getInt64(DeviceID),
3945 Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP,
3946 ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
3947}
3948
3949bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
3950 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
3951 assert(!(AO == AtomicOrdering::NotAtomic ||(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3953, __extension__
__PRETTY_FUNCTION__))
3952 AO == llvm::AtomicOrdering::Unordered) &&(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3953, __extension__
__PRETTY_FUNCTION__))
3953 "Unexpected Atomic Ordering.")(static_cast <bool> (!(AO == AtomicOrdering::NotAtomic ||
AO == llvm::AtomicOrdering::Unordered) && "Unexpected Atomic Ordering."
) ? void (0) : __assert_fail ("!(AO == AtomicOrdering::NotAtomic || AO == llvm::AtomicOrdering::Unordered) && \"Unexpected Atomic Ordering.\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 3953, __extension__
__PRETTY_FUNCTION__))
;
3954
3955 bool Flush = false;
3956 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
3957
3958 switch (AK) {
3959 case Read:
3960 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
3961 AO == AtomicOrdering::SequentiallyConsistent) {
3962 FlushAO = AtomicOrdering::Acquire;
3963 Flush = true;
3964 }
3965 break;
3966 case Write:
3967 case Compare:
3968 case Update:
3969 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
3970 AO == AtomicOrdering::SequentiallyConsistent) {
3971 FlushAO = AtomicOrdering::Release;
3972 Flush = true;
3973 }
3974 break;
3975 case Capture:
3976 switch (AO) {
3977 case AtomicOrdering::Acquire:
3978 FlushAO = AtomicOrdering::Acquire;
3979 Flush = true;
3980 break;
3981 case AtomicOrdering::Release:
3982 FlushAO = AtomicOrdering::Release;
3983 Flush = true;
3984 break;
3985 case AtomicOrdering::AcquireRelease:
3986 case AtomicOrdering::SequentiallyConsistent:
3987 FlushAO = AtomicOrdering::AcquireRelease;
3988 Flush = true;
3989 break;
3990 default:
3991 // do nothing - leave silently.
3992 break;
3993 }
3994 }
3995
3996 if (Flush) {
3997 // Currently Flush RT call still doesn't take memory_ordering, so for when
3998 // that happens, this tries to do the resolution of which atomic ordering
3999 // to use with but issue the flush call
4000 // TODO: pass `FlushAO` after memory ordering support is added
4001 (void)FlushAO;
4002 emitFlush(Loc);
4003 }
4004
4005 // for AO == AtomicOrdering::Monotonic and all other case combinations
4006 // do nothing
4007 return Flush;
4008}
4009
4010OpenMPIRBuilder::InsertPointTy
4011OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
4012 AtomicOpValue &X, AtomicOpValue &V,
4013 AtomicOrdering AO) {
4014 if (!updateToLocation(Loc))
4015 return Loc.IP;
4016
4017 Type *XTy = X.Var->getType();
4018 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4018, __extension__
__PRETTY_FUNCTION__))
;
4019 Type *XElemTy = X.ElemTy;
4020 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4022, __extension__
__PRETTY_FUNCTION__))
4021 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4022, __extension__
__PRETTY_FUNCTION__))
4022 "OMP atomic read expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic read expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic read expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4022, __extension__
__PRETTY_FUNCTION__))
;
4023
4024 Value *XRead = nullptr;
4025
4026 if (XElemTy->isIntegerTy()) {
4027 LoadInst *XLD =
4028 Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");
4029 XLD->setAtomic(AO);
4030 XRead = cast<Value>(XLD);
4031 } else {
4032 // We need to bitcast and perform atomic op as integer
4033 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
4034 IntegerType *IntCastTy =
4035 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4036 Value *XBCast = Builder.CreateBitCast(
4037 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.src.int.cast");
4038 LoadInst *XLoad =
4039 Builder.CreateLoad(IntCastTy, XBCast, X.IsVolatile, "omp.atomic.load");
4040 XLoad->setAtomic(AO);
4041 if (XElemTy->isFloatingPointTy()) {
4042 XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");
4043 } else {
4044 XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");
4045 }
4046 }
4047 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
4048 Builder.CreateStore(XRead, V.Var, V.IsVolatile);
4049 return Builder.saveIP();
4050}
4051
4052OpenMPIRBuilder::InsertPointTy
4053OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
4054 AtomicOpValue &X, Value *Expr,
4055 AtomicOrdering AO) {
4056 if (!updateToLocation(Loc))
4057 return Loc.IP;
4058
4059 Type *XTy = X.Var->getType();
4060 assert(XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory")(static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4060, __extension__
__PRETTY_FUNCTION__))
;
4061 Type *XElemTy = X.ElemTy;
4062 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4064, __extension__
__PRETTY_FUNCTION__))
4063 XElemTy->isPointerTy()) &&(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4064, __extension__
__PRETTY_FUNCTION__))
4064 "OMP atomic write expected a scalar type")(static_cast <bool> ((XElemTy->isFloatingPointTy() ||
XElemTy->isIntegerTy() || XElemTy->isPointerTy()) &&
"OMP atomic write expected a scalar type") ? void (0) : __assert_fail
("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic write expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4064, __extension__
__PRETTY_FUNCTION__))
;
4065
4066 if (XElemTy->isIntegerTy()) {
4067 StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);
4068 XSt->setAtomic(AO);
4069 } else {
4070 // We need to bitcast and perform atomic op as integers
4071 unsigned Addrspace = cast<PointerType>(XTy)->getAddressSpace();
4072 IntegerType *IntCastTy =
4073 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4074 Value *XBCast = Builder.CreateBitCast(
4075 X.Var, IntCastTy->getPointerTo(Addrspace), "atomic.dst.int.cast");
4076 Value *ExprCast =
4077 Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");
4078 StoreInst *XSt = Builder.CreateStore(ExprCast, XBCast, X.IsVolatile);
4079 XSt->setAtomic(AO);
4080 }
4081
4082 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
4083 return Builder.saveIP();
4084}
4085
4086OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
4087 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
4088 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
4089 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
4090 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous")(static_cast <bool> (!isConflictIP(Loc.IP, AllocaIP) &&
"IPs must not be ambiguous") ? void (0) : __assert_fail ("!isConflictIP(Loc.IP, AllocaIP) && \"IPs must not be ambiguous\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4090, __extension__
__PRETTY_FUNCTION__))
;
1
'?' condition is true
4091 if (!updateToLocation(Loc))
4092 return Loc.IP;
4093
4094 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
2
Taking false branch
3
Assuming 'DebugFlag' is false
4
Loop condition is false. Exiting loop
4095 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4096 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4097 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4098 Type *XElemTy = X.ElemTy;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4099 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4100 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4101 "OMP atomic update expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4102 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4103 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4104 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4105 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4097, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic update expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic update expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4101, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp
!= AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::
UMin) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4104, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
4106
4107 emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
5
Calling 'OpenMPIRBuilder::emitAtomicUpdate'
4108 X.IsVolatile, IsXBinopExpr);
4109 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
4110 return Builder.saveIP();
4111}
4112
4113Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
4114 AtomicRMWInst::BinOp RMWOp) {
4115 switch (RMWOp) {
4116 case AtomicRMWInst::Add:
4117 return Builder.CreateAdd(Src1, Src2);
4118 case AtomicRMWInst::Sub:
4119 return Builder.CreateSub(Src1, Src2);
4120 case AtomicRMWInst::And:
4121 return Builder.CreateAnd(Src1, Src2);
4122 case AtomicRMWInst::Nand:
4123 return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));
4124 case AtomicRMWInst::Or:
4125 return Builder.CreateOr(Src1, Src2);
4126 case AtomicRMWInst::Xor:
4127 return Builder.CreateXor(Src1, Src2);
4128 case AtomicRMWInst::Xchg:
4129 case AtomicRMWInst::FAdd:
4130 case AtomicRMWInst::FSub:
4131 case AtomicRMWInst::BAD_BINOP:
4132 case AtomicRMWInst::Max:
4133 case AtomicRMWInst::Min:
4134 case AtomicRMWInst::UMax:
4135 case AtomicRMWInst::UMin:
4136 case AtomicRMWInst::FMax:
4137 case AtomicRMWInst::FMin:
4138 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4138)
;
4139 }
4140 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4140)
;
4141}
4142
4143std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
4144 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
4145 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
4146 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
4147 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
4148 // or a complex datatype.
4149 bool emitRMWOp = false;
4150 switch (RMWOp) {
6
Control jumps to the 'default' case at line 4162
4151 case AtomicRMWInst::Add:
4152 case AtomicRMWInst::And:
4153 case AtomicRMWInst::Nand:
4154 case AtomicRMWInst::Or:
4155 case AtomicRMWInst::Xor:
4156 case AtomicRMWInst::Xchg:
4157 emitRMWOp = XElemTy;
4158 break;
4159 case AtomicRMWInst::Sub:
4160 emitRMWOp = (IsXBinopExpr && XElemTy);
4161 break;
4162 default:
4163 emitRMWOp = false;
4164 }
4165 emitRMWOp &= XElemTy->isIntegerTy();
7
Calling 'Type::isIntegerTy'
10
Returning from 'Type::isIntegerTy'
4166
4167 std::pair<Value *, Value *> Res;
4168 if (emitRMWOp
10.1
'emitRMWOp' is false
10.1
'emitRMWOp' is false
10.1
'emitRMWOp' is false
) {
11
Taking false branch
4169 Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
4170 // not needed except in case of postfix captures. Generate anyway for
4171 // consistency with the else part. Will be removed with any DCE pass.
4172 // AtomicRMWInst::Xchg does not have a coressponding instruction.
4173 if (RMWOp == AtomicRMWInst::Xchg)
4174 Res.second = Res.first;
4175 else
4176 Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
4177 } else {
4178 unsigned Addrspace = cast<PointerType>(X->getType())->getAddressSpace();
12
The object is a 'CastReturnType'
4179 IntegerType *IntCastTy =
4180 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4181 Value *XBCast =
4182 Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
4183 LoadInst *OldVal =
4184 Builder.CreateLoad(IntCastTy, XBCast, X->getName() + ".atomic.load");
4185 OldVal->setAtomic(AO);
4186 // CurBB
4187 // | /---\
4188 // ContBB |
4189 // | \---/
4190 // ExitBB
4191 BasicBlock *CurBB = Builder.GetInsertBlock();
4192 Instruction *CurBBTI = CurBB->getTerminator();
4193 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
13
Assuming 'CurBBTI' is null
14
'?' condition is false
4194 BasicBlock *ExitBB =
4195 CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
4196 BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
4197 X->getName() + ".atomic.cont");
4198 ContBB->getTerminator()->eraseFromParent();
4199 Builder.restoreIP(AllocaIP);
4200 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
4201 NewAtomicAddr->setName(X->getName() + "x.new.val");
4202 Builder.SetInsertPoint(ContBB);
4203 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
4204 PHI->addIncoming(OldVal, CurBB);
4205 IntegerType *NewAtomicCastTy =
4206 IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
4207 bool IsIntTy = XElemTy->isIntegerTy();
4208 Value *NewAtomicIntAddr =
4209 (IsIntTy
14.1
'IsIntTy' is false
14.1
'IsIntTy' is false
14.1
'IsIntTy' is false
)
15
'?' condition is false
4210 ? NewAtomicAddr
4211 : Builder.CreateBitCast(NewAtomicAddr,
4212 NewAtomicCastTy->getPointerTo(Addrspace));
4213 Value *OldExprVal = PHI;
4214 if (!IsIntTy
15.1
'IsIntTy' is false
15.1
'IsIntTy' is false
15.1
'IsIntTy' is false
) {
16
Taking true branch
4215 if (XElemTy->isFloatingPointTy()) {
17
Taking true branch
4216 OldExprVal = Builder.CreateBitCast(PHI, XElemTy,
4217 X->getName() + ".atomic.fltCast");
4218 } else {
4219 OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,
4220 X->getName() + ".atomic.ptrCast");
4221 }
4222 }
4223
4224 Value *Upd = UpdateOp(OldExprVal, Builder);
4225 Builder.CreateStore(Upd, NewAtomicAddr);
4226 LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicIntAddr);
4227 Value *XAddr =
4228 (IsIntTy
17.1
'IsIntTy' is false
17.1
'IsIntTy' is false
17.1
'IsIntTy' is false
)
18
'?' condition is false
4229 ? X
4230 : Builder.CreateBitCast(X, IntCastTy->getPointerTo(Addrspace));
4231 AtomicOrdering Failure =
4232 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
4233 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
4234 XAddr, PHI, DesiredVal, llvm::MaybeAlign(), AO, Failure);
4235 Result->setVolatile(VolatileX);
4236 Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);
4237 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4238 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());
4239 Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);
4240
4241 Res.first = OldExprVal;
4242 Res.second = Upd;
4243
4244 // set Insertion point in exit block
4245 if (UnreachableInst *ExitTI
20.1
'ExitTI' is null
20.1
'ExitTI' is null
20.1
'ExitTI' is null
=
20
'ExitTI' initialized to a null pointer value
21
Taking false branch
4246 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
19
Assuming the object is not a 'CastReturnType'
4247 CurBBTI->eraseFromParent();
4248 Builder.SetInsertPoint(ExitBB);
4249 } else {
4250 Builder.SetInsertPoint(ExitTI);
22
Passing null pointer value via 1st parameter 'I'
23
Calling 'IRBuilderBase::SetInsertPoint'
4251 }
4252 }
4253
4254 return Res;
4255}
4256
4257OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
4258 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
4259 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
4260 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
4261 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
4262 if (!updateToLocation(Loc))
4263 return Loc.IP;
4264
4265 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4266 Type *XTy = X.Var->getType();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4267 assert(XTy->isPointerTy() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4268 "OMP Atomic expects a pointer to target memory");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4269 Type *XElemTy = X.ElemTy;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4270 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4271 XElemTy->isPointerTy()) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4272 "OMP atomic capture expected a scalar type");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4273 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4274 "OpenMP atomic does not support LT or GT operations");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
4275 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("openmp-ir-builder")) { { Type *XTy = X.Var->getType(); (
static_cast <bool> (XTy->isPointerTy() && "OMP Atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("XTy->isPointerTy() && \"OMP Atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4268, __extension__
__PRETTY_FUNCTION__)); Type *XElemTy = X.ElemTy; (static_cast
<bool> ((XElemTy->isFloatingPointTy() || XElemTy->
isIntegerTy() || XElemTy->isPointerTy()) && "OMP atomic capture expected a scalar type"
) ? void (0) : __assert_fail ("(XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() || XElemTy->isPointerTy()) && \"OMP atomic capture expected a scalar type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4272, __extension__
__PRETTY_FUNCTION__)); (static_cast <bool> ((RMWOp != AtomicRMWInst
::Max) && (RMWOp != AtomicRMWInst::Min) && "OpenMP atomic does not support LT or GT operations"
) ? void (0) : __assert_fail ("(RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && \"OpenMP atomic does not support LT or GT operations\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4274, __extension__
__PRETTY_FUNCTION__)); }; } } while (false)
;
4276
4277 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
4278 // 'x' is simply atomically rewritten with 'expr'.
4279 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
4280 std::pair<Value *, Value *> Result =
4281 emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
4282 X.IsVolatile, IsXBinopExpr);
4283
4284 Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
4285 Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
4286
4287 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
4288 return Builder.saveIP();
4289}
4290
4291OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
4292 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
4293 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
4294 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
4295 bool IsFailOnly) {
4296
4297 if (!updateToLocation(Loc))
4298 return Loc.IP;
4299
4300 assert(X.Var->getType()->isPointerTy() &&(static_cast <bool> (X.Var->getType()->isPointerTy
() && "OMP atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("X.Var->getType()->isPointerTy() && \"OMP atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4301, __extension__
__PRETTY_FUNCTION__))
4301 "OMP atomic expects a pointer to target memory")(static_cast <bool> (X.Var->getType()->isPointerTy
() && "OMP atomic expects a pointer to target memory"
) ? void (0) : __assert_fail ("X.Var->getType()->isPointerTy() && \"OMP atomic expects a pointer to target memory\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4301, __extension__
__PRETTY_FUNCTION__))
;
4302 // compare capture
4303 if (V.Var) {
4304 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type")(static_cast <bool> (V.Var->getType()->isPointerTy
() && "v.var must be of pointer type") ? void (0) : __assert_fail
("V.Var->getType()->isPointerTy() && \"v.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4304, __extension__
__PRETTY_FUNCTION__))
;
4305 assert(V.ElemTy == X.ElemTy && "x and v must be of same type")(static_cast <bool> (V.ElemTy == X.ElemTy && "x and v must be of same type"
) ? void (0) : __assert_fail ("V.ElemTy == X.ElemTy && \"x and v must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4305, __extension__
__PRETTY_FUNCTION__))
;
4306 }
4307
4308 bool IsInteger = E->getType()->isIntegerTy();
4309
4310 if (Op == OMPAtomicCompareOp::EQ) {
4311 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
4312 AtomicCmpXchgInst *Result = nullptr;
4313 if (!IsInteger) {
4314 unsigned Addrspace =
4315 cast<PointerType>(X.Var->getType())->getAddressSpace();
4316 IntegerType *IntCastTy =
4317 IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits());
4318 Value *XBCast =
4319 Builder.CreateBitCast(X.Var, IntCastTy->getPointerTo(Addrspace));
4320 Value *EBCast = Builder.CreateBitCast(E, IntCastTy);
4321 Value *DBCast = Builder.CreateBitCast(D, IntCastTy);
4322 Result = Builder.CreateAtomicCmpXchg(XBCast, EBCast, DBCast, MaybeAlign(),
4323 AO, Failure);
4324 } else {
4325 Result =
4326 Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);
4327 }
4328
4329 if (V.Var) {
4330 Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0);
4331 if (!IsInteger)
4332 OldValue = Builder.CreateBitCast(OldValue, X.ElemTy);
4333 assert(OldValue->getType() == V.ElemTy &&(static_cast <bool> (OldValue->getType() == V.ElemTy
&& "OldValue and V must be of same type") ? void (0)
: __assert_fail ("OldValue->getType() == V.ElemTy && \"OldValue and V must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4334, __extension__
__PRETTY_FUNCTION__))
4334 "OldValue and V must be of same type")(static_cast <bool> (OldValue->getType() == V.ElemTy
&& "OldValue and V must be of same type") ? void (0)
: __assert_fail ("OldValue->getType() == V.ElemTy && \"OldValue and V must be of same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4334, __extension__
__PRETTY_FUNCTION__))
;
4335 if (IsPostfixUpdate) {
4336 Builder.CreateStore(OldValue, V.Var, V.IsVolatile);
4337 } else {
4338 Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4339 if (IsFailOnly) {
4340 // CurBB----
4341 // | |
4342 // v |
4343 // ContBB |
4344 // | |
4345 // v |
4346 // ExitBB <-
4347 //
4348 // where ContBB only contains the store of old value to 'v'.
4349 BasicBlock *CurBB = Builder.GetInsertBlock();
4350 Instruction *CurBBTI = CurBB->getTerminator();
4351 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
4352 BasicBlock *ExitBB = CurBB->splitBasicBlock(
4353 CurBBTI, X.Var->getName() + ".atomic.exit");
4354 BasicBlock *ContBB = CurBB->splitBasicBlock(
4355 CurBB->getTerminator(), X.Var->getName() + ".atomic.cont");
4356 ContBB->getTerminator()->eraseFromParent();
4357 CurBB->getTerminator()->eraseFromParent();
4358
4359 Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB);
4360
4361 Builder.SetInsertPoint(ContBB);
4362 Builder.CreateStore(OldValue, V.Var);
4363 Builder.CreateBr(ExitBB);
4364
4365 if (UnreachableInst *ExitTI =
4366 dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
4367 CurBBTI->eraseFromParent();
4368 Builder.SetInsertPoint(ExitBB);
4369 } else {
4370 Builder.SetInsertPoint(ExitTI);
4371 }
4372 } else {
4373 Value *CapturedValue =
4374 Builder.CreateSelect(SuccessOrFail, E, OldValue);
4375 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
4376 }
4377 }
4378 }
4379 // The comparison result has to be stored.
4380 if (R.Var) {
4381 assert(R.Var->getType()->isPointerTy() &&(static_cast <bool> (R.Var->getType()->isPointerTy
() && "r.var must be of pointer type") ? void (0) : __assert_fail
("R.Var->getType()->isPointerTy() && \"r.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4382, __extension__
__PRETTY_FUNCTION__))
4382 "r.var must be of pointer type")(static_cast <bool> (R.Var->getType()->isPointerTy
() && "r.var must be of pointer type") ? void (0) : __assert_fail
("R.Var->getType()->isPointerTy() && \"r.var must be of pointer type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4382, __extension__
__PRETTY_FUNCTION__))
;
4383 assert(R.ElemTy->isIntegerTy() && "r must be of integral type")(static_cast <bool> (R.ElemTy->isIntegerTy() &&
"r must be of integral type") ? void (0) : __assert_fail ("R.ElemTy->isIntegerTy() && \"r must be of integral type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4383, __extension__
__PRETTY_FUNCTION__))
;
4384
4385 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
4386 Value *ResultCast = R.IsSigned
4387 ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy)
4388 : Builder.CreateZExt(SuccessFailureVal, R.ElemTy);
4389 Builder.CreateStore(ResultCast, R.Var, R.IsVolatile);
4390 }
4391 } else {
4392 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&(static_cast <bool> ((Op == OMPAtomicCompareOp::MAX || Op
== OMPAtomicCompareOp::MIN) && "Op should be either max or min at this point"
) ? void (0) : __assert_fail ("(Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) && \"Op should be either max or min at this point\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4393, __extension__
__PRETTY_FUNCTION__))
4393 "Op should be either max or min at this point")(static_cast <bool> ((Op == OMPAtomicCompareOp::MAX || Op
== OMPAtomicCompareOp::MIN) && "Op should be either max or min at this point"
) ? void (0) : __assert_fail ("(Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) && \"Op should be either max or min at this point\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4393, __extension__
__PRETTY_FUNCTION__))
;
4394 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==")(static_cast <bool> (!IsFailOnly && "IsFailOnly is only valid when the comparison is =="
) ? void (0) : __assert_fail ("!IsFailOnly && \"IsFailOnly is only valid when the comparison is ==\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4394, __extension__
__PRETTY_FUNCTION__))
;
4395
4396 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
4397 // Let's take max as example.
4398 // OpenMP form:
4399 // x = x > expr ? expr : x;
4400 // LLVM form:
4401 // *ptr = *ptr > val ? *ptr : val;
4402 // We need to transform to LLVM form.
4403 // x = x <= expr ? x : expr;
4404 AtomicRMWInst::BinOp NewOp;
4405 if (IsXBinopExpr) {
4406 if (IsInteger) {
4407 if (X.IsSigned)
4408 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
4409 : AtomicRMWInst::Max;
4410 else
4411 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
4412 : AtomicRMWInst::UMax;
4413 } else {
4414 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
4415 : AtomicRMWInst::FMax;
4416 }
4417 } else {
4418 if (IsInteger) {
4419 if (X.IsSigned)
4420 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
4421 : AtomicRMWInst::Min;
4422 else
4423 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
4424 : AtomicRMWInst::UMin;
4425 } else {
4426 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
4427 : AtomicRMWInst::FMin;
4428 }
4429 }
4430
4431 AtomicRMWInst *OldValue =
4432 Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);
4433 if (V.Var) {
4434 Value *CapturedValue = nullptr;
4435 if (IsPostfixUpdate) {
4436 CapturedValue = OldValue;
4437 } else {
4438 CmpInst::Predicate Pred;
4439 switch (NewOp) {
4440 case AtomicRMWInst::Max:
4441 Pred = CmpInst::ICMP_SGT;
4442 break;
4443 case AtomicRMWInst::UMax:
4444 Pred = CmpInst::ICMP_UGT;
4445 break;
4446 case AtomicRMWInst::FMax:
4447 Pred = CmpInst::FCMP_OGT;
4448 break;
4449 case AtomicRMWInst::Min:
4450 Pred = CmpInst::ICMP_SLT;
4451 break;
4452 case AtomicRMWInst::UMin:
4453 Pred = CmpInst::ICMP_ULT;
4454 break;
4455 case AtomicRMWInst::FMin:
4456 Pred = CmpInst::FCMP_OLT;
4457 break;
4458 default:
4459 llvm_unreachable("unexpected comparison op")::llvm::llvm_unreachable_internal("unexpected comparison op",
"llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4459)
;
4460 }
4461 Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E);
4462 CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue);
4463 }
4464 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
4465 }
4466 }
4467
4468 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);
4469
4470 return Builder.saveIP();
4471}
4472
4473GlobalVariable *
4474OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
4475 std::string VarName) {
4476 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
4477 llvm::ArrayType::get(
4478 llvm::Type::getInt8Ty(M.getContext())->getPointerTo(), Names.size()),
4479 Names);
4480 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
4481 M, MapNamesArrayInit->getType(),
4482 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
4483 VarName);
4484 return MapNamesArrayGlobal;
4485}
4486
4487// Create all simple and struct types exposed by the runtime and remember
4488// the llvm::PointerTypes of them for easy access later.
4489void OpenMPIRBuilder::initializeTypes(Module &M) {
4490 LLVMContext &Ctx = M.getContext();
4491 StructType *T;
4492#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
4493#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
4494 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
4495 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
4496#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
4497 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
4498 VarName##Ptr = PointerType::getUnqual(VarName);
4499#define OMP_STRUCT_TYPE(VarName, StructName, ...) \
4500 T = StructType::getTypeByName(Ctx, StructName); \
4501 if (!T) \
4502 T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
4503 VarName = T; \
4504 VarName##Ptr = PointerType::getUnqual(T);
4505#include "llvm/Frontend/OpenMP/OMPKinds.def"
4506}
4507
4508void OpenMPIRBuilder::OutlineInfo::collectBlocks(
4509 SmallPtrSetImpl<BasicBlock *> &BlockSet,
4510 SmallVectorImpl<BasicBlock *> &BlockVector) {
4511 SmallVector<BasicBlock *, 32> Worklist;
4512 BlockSet.insert(EntryBB);
4513 BlockSet.insert(ExitBB);
4514
4515 Worklist.push_back(EntryBB);
4516 while (!Worklist.empty()) {
4517 BasicBlock *BB = Worklist.pop_back_val();
4518 BlockVector.push_back(BB);
4519 for (BasicBlock *SuccBB : successors(BB))
4520 if (BlockSet.insert(SuccBB).second)
4521 Worklist.push_back(SuccBB);
4522 }
4523}
4524
4525void CanonicalLoopInfo::collectControlBlocks(
4526 SmallVectorImpl<BasicBlock *> &BBs) {
4527 // We only count those BBs as control block for which we do not need to
4528 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
4529 // flow. For consistency, this also means we do not add the Body block, which
4530 // is just the entry to the body code.
4531 BBs.reserve(BBs.size() + 6);
4532 BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});
4533}
4534
4535BasicBlock *CanonicalLoopInfo::getPreheader() const {
4536 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4536, __extension__
__PRETTY_FUNCTION__))
;
4537 for (BasicBlock *Pred : predecessors(Header)) {
4538 if (Pred != Latch)
4539 return Pred;
4540 }
4541 llvm_unreachable("Missing preheader")::llvm::llvm_unreachable_internal("Missing preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp"
, 4541)
;
4542}
4543
4544void CanonicalLoopInfo::setTripCount(Value *TripCount) {
4545 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4545, __extension__
__PRETTY_FUNCTION__))
;
4546
4547 Instruction *CmpI = &getCond()->front();
4548 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount")(static_cast <bool> (isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount") ? void (0) : __assert_fail
("isa<CmpInst>(CmpI) && \"First inst must compare IV with TripCount\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4548, __extension__
__PRETTY_FUNCTION__))
;
4549 CmpI->setOperand(1, TripCount);
4550
4551#ifndef NDEBUG
4552 assertOK();
4553#endif
4554}
4555
4556void CanonicalLoopInfo::mapIndVar(
4557 llvm::function_ref<Value *(Instruction *)> Updater) {
4558 assert(isValid() && "Requires a valid canonical loop")(static_cast <bool> (isValid() && "Requires a valid canonical loop"
) ? void (0) : __assert_fail ("isValid() && \"Requires a valid canonical loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4558, __extension__
__PRETTY_FUNCTION__))
;
4559
4560 Instruction *OldIV = getIndVar();
4561
4562 // Record all uses excluding those introduced by the updater. Uses by the
4563 // CanonicalLoopInfo itself to keep track of the number of iterations are
4564 // excluded.
4565 SmallVector<Use *> ReplacableUses;
4566 for (Use &U : OldIV->uses()) {
4567 auto *User = dyn_cast<Instruction>(U.getUser());
4568 if (!User)
4569 continue;
4570 if (User->getParent() == getCond())
4571 continue;
4572 if (User->getParent() == getLatch())
4573 continue;
4574 ReplacableUses.push_back(&U);
4575 }
4576
4577 // Run the updater that may introduce new uses
4578 Value *NewIV = Updater(OldIV);
4579
4580 // Replace the old uses with the value returned by the updater.
4581 for (Use *U : ReplacableUses)
4582 U->set(NewIV);
4583
4584#ifndef NDEBUG
4585 assertOK();
4586#endif
4587}
4588
4589void CanonicalLoopInfo::assertOK() const {
4590#ifndef NDEBUG
4591 // No constraints if this object currently does not describe a loop.
4592 if (!isValid())
4593 return;
4594
4595 BasicBlock *Preheader = getPreheader();
4596 BasicBlock *Body = getBody();
4597 BasicBlock *After = getAfter();
4598
4599 // Verify standard control-flow we use for OpenMP loops.
4600 assert(Preheader)(static_cast <bool> (Preheader) ? void (0) : __assert_fail
("Preheader", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4600
, __extension__ __PRETTY_FUNCTION__))
;
4601 assert(isa<BranchInst>(Preheader->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4602, __extension__
__PRETTY_FUNCTION__))
4602 "Preheader must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Preheader->
getTerminator()) && "Preheader must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Preheader->getTerminator()) && \"Preheader must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4602, __extension__
__PRETTY_FUNCTION__))
;
4603 assert(Preheader->getSingleSuccessor() == Header &&(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4604, __extension__
__PRETTY_FUNCTION__))
4604 "Preheader must jump to header")(static_cast <bool> (Preheader->getSingleSuccessor()
== Header && "Preheader must jump to header") ? void
(0) : __assert_fail ("Preheader->getSingleSuccessor() == Header && \"Preheader must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4604, __extension__
__PRETTY_FUNCTION__))
;
4605
4606 assert(Header)(static_cast <bool> (Header) ? void (0) : __assert_fail
("Header", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4606
, __extension__ __PRETTY_FUNCTION__))
;
4607 assert(isa<BranchInst>(Header->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4608, __extension__
__PRETTY_FUNCTION__))
4608 "Header must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Header->getTerminator
()) && "Header must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Header->getTerminator()) && \"Header must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4608, __extension__
__PRETTY_FUNCTION__))
;
4609 assert(Header->getSingleSuccessor() == Cond &&(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4610, __extension__
__PRETTY_FUNCTION__))
4610 "Header must jump to exiting block")(static_cast <bool> (Header->getSingleSuccessor() ==
Cond && "Header must jump to exiting block") ? void (
0) : __assert_fail ("Header->getSingleSuccessor() == Cond && \"Header must jump to exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4610, __extension__
__PRETTY_FUNCTION__))
;
4611
4612 assert(Cond)(static_cast <bool> (Cond) ? void (0) : __assert_fail (
"Cond", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4612, __extension__
__PRETTY_FUNCTION__))
;
4613 assert(Cond->getSinglePredecessor() == Header &&(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4614, __extension__
__PRETTY_FUNCTION__))
4614 "Exiting block only reachable from header")(static_cast <bool> (Cond->getSinglePredecessor() ==
Header && "Exiting block only reachable from header"
) ? void (0) : __assert_fail ("Cond->getSinglePredecessor() == Header && \"Exiting block only reachable from header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4614, __extension__
__PRETTY_FUNCTION__))
;
4615
4616 assert(isa<BranchInst>(Cond->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4617, __extension__
__PRETTY_FUNCTION__))
4617 "Exiting block must terminate with conditional branch")(static_cast <bool> (isa<BranchInst>(Cond->getTerminator
()) && "Exiting block must terminate with conditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Cond->getTerminator()) && \"Exiting block must terminate with conditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4617, __extension__
__PRETTY_FUNCTION__))
;
4618 assert(size(successors(Cond)) == 2 &&(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4619, __extension__
__PRETTY_FUNCTION__))
4619 "Exiting block must have two successors")(static_cast <bool> (size(successors(Cond)) == 2 &&
"Exiting block must have two successors") ? void (0) : __assert_fail
("size(successors(Cond)) == 2 && \"Exiting block must have two successors\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4619, __extension__
__PRETTY_FUNCTION__))
;
4620 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4621, __extension__
__PRETTY_FUNCTION__))
4621 "Exiting block's first successor jump to the body")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(0) == Body && "Exiting block's first successor jump to the body"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body && \"Exiting block's first successor jump to the body\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4621, __extension__
__PRETTY_FUNCTION__))
;
4622 assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4623, __extension__
__PRETTY_FUNCTION__))
4623 "Exiting block's second successor must exit the loop")(static_cast <bool> (cast<BranchInst>(Cond->getTerminator
())->getSuccessor(1) == Exit && "Exiting block's second successor must exit the loop"
) ? void (0) : __assert_fail ("cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit && \"Exiting block's second successor must exit the loop\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4623, __extension__
__PRETTY_FUNCTION__))
;
4624
4625 assert(Body)(static_cast <bool> (Body) ? void (0) : __assert_fail (
"Body", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4625, __extension__
__PRETTY_FUNCTION__))
;
4626 assert(Body->getSinglePredecessor() == Cond &&(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4627, __extension__
__PRETTY_FUNCTION__))
4627 "Body only reachable from exiting block")(static_cast <bool> (Body->getSinglePredecessor() ==
Cond && "Body only reachable from exiting block") ? void
(0) : __assert_fail ("Body->getSinglePredecessor() == Cond && \"Body only reachable from exiting block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4627, __extension__
__PRETTY_FUNCTION__))
;
4628 assert(!isa<PHINode>(Body->front()))(static_cast <bool> (!isa<PHINode>(Body->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Body->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4628, __extension__
__PRETTY_FUNCTION__))
;
4629
4630 assert(Latch)(static_cast <bool> (Latch) ? void (0) : __assert_fail (
"Latch", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4630, __extension__
__PRETTY_FUNCTION__))
;
4631 assert(isa<BranchInst>(Latch->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4632, __extension__
__PRETTY_FUNCTION__))
4632 "Latch must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Latch->getTerminator
()) && "Latch must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Latch->getTerminator()) && \"Latch must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4632, __extension__
__PRETTY_FUNCTION__))
;
4633 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header")(static_cast <bool> (Latch->getSingleSuccessor() == Header
&& "Latch must jump to header") ? void (0) : __assert_fail
("Latch->getSingleSuccessor() == Header && \"Latch must jump to header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4633, __extension__
__PRETTY_FUNCTION__))
;
4634 // TODO: To support simple redirecting of the end of the body code that has
4635 // multiple; introduce another auxiliary basic block like preheader and after.
4636 assert(Latch->getSinglePredecessor() != nullptr)(static_cast <bool> (Latch->getSinglePredecessor() !=
nullptr) ? void (0) : __assert_fail ("Latch->getSinglePredecessor() != nullptr"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4636, __extension__
__PRETTY_FUNCTION__))
;
4637 assert(!isa<PHINode>(Latch->front()))(static_cast <bool> (!isa<PHINode>(Latch->front
())) ? void (0) : __assert_fail ("!isa<PHINode>(Latch->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4637, __extension__
__PRETTY_FUNCTION__))
;
4638
4639 assert(Exit)(static_cast <bool> (Exit) ? void (0) : __assert_fail (
"Exit", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4639, __extension__
__PRETTY_FUNCTION__))
;
4640 assert(isa<BranchInst>(Exit->getTerminator()) &&(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4641, __extension__
__PRETTY_FUNCTION__))
4641 "Exit block must terminate with unconditional branch")(static_cast <bool> (isa<BranchInst>(Exit->getTerminator
()) && "Exit block must terminate with unconditional branch"
) ? void (0) : __assert_fail ("isa<BranchInst>(Exit->getTerminator()) && \"Exit block must terminate with unconditional branch\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4641, __extension__
__PRETTY_FUNCTION__))
;
4642 assert(Exit->getSingleSuccessor() == After &&(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4643, __extension__
__PRETTY_FUNCTION__))
4643 "Exit block must jump to after block")(static_cast <bool> (Exit->getSingleSuccessor() == After
&& "Exit block must jump to after block") ? void (0)
: __assert_fail ("Exit->getSingleSuccessor() == After && \"Exit block must jump to after block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4643, __extension__
__PRETTY_FUNCTION__))
;
4644
4645 assert(After)(static_cast <bool> (After) ? void (0) : __assert_fail (
"After", "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4645, __extension__
__PRETTY_FUNCTION__))
;
4646 assert(After->getSinglePredecessor() == Exit &&(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4647, __extension__
__PRETTY_FUNCTION__))
4647 "After block only reachable from exit block")(static_cast <bool> (After->getSinglePredecessor() ==
Exit && "After block only reachable from exit block"
) ? void (0) : __assert_fail ("After->getSinglePredecessor() == Exit && \"After block only reachable from exit block\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4647, __extension__
__PRETTY_FUNCTION__))
;
4648 assert(After->empty() || !isa<PHINode>(After->front()))(static_cast <bool> (After->empty() || !isa<PHINode
>(After->front())) ? void (0) : __assert_fail ("After->empty() || !isa<PHINode>(After->front())"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4648, __extension__
__PRETTY_FUNCTION__))
;
4649
4650 Instruction *IndVar = getIndVar();
4651 assert(IndVar && "Canonical induction variable not found?")(static_cast <bool> (IndVar && "Canonical induction variable not found?"
) ? void (0) : __assert_fail ("IndVar && \"Canonical induction variable not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4651, __extension__
__PRETTY_FUNCTION__))
;
4652 assert(isa<IntegerType>(IndVar->getType()) &&(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4653, __extension__
__PRETTY_FUNCTION__))
4653 "Induction variable must be an integer")(static_cast <bool> (isa<IntegerType>(IndVar->
getType()) && "Induction variable must be an integer"
) ? void (0) : __assert_fail ("isa<IntegerType>(IndVar->getType()) && \"Induction variable must be an integer\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4653, __extension__
__PRETTY_FUNCTION__))
;
4654 assert(cast<PHINode>(IndVar)->getParent() == Header &&(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4655, __extension__
__PRETTY_FUNCTION__))
4655 "Induction variable must be a PHI in the loop header")(static_cast <bool> (cast<PHINode>(IndVar)->getParent
() == Header && "Induction variable must be a PHI in the loop header"
) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getParent() == Header && \"Induction variable must be a PHI in the loop header\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4655, __extension__
__PRETTY_FUNCTION__))
;
4656 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(0) == Preheader) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4656, __extension__
__PRETTY_FUNCTION__))
;
4657 assert((static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4658, __extension__
__PRETTY_FUNCTION__))
4658 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero())(static_cast <bool> (cast<ConstantInt>(cast<PHINode
>(IndVar)->getIncomingValue(0))->isZero()) ? void (0
) : __assert_fail ("cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4658, __extension__
__PRETTY_FUNCTION__))
;
4659 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch)(static_cast <bool> (cast<PHINode>(IndVar)->getIncomingBlock
(1) == Latch) ? void (0) : __assert_fail ("cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4659, __extension__
__PRETTY_FUNCTION__))
;
4660
4661 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
4662 assert(cast<Instruction>(NextIndVar)->getParent() == Latch)(static_cast <bool> (cast<Instruction>(NextIndVar
)->getParent() == Latch) ? void (0) : __assert_fail ("cast<Instruction>(NextIndVar)->getParent() == Latch"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4662, __extension__
__PRETTY_FUNCTION__))
;
4663 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOpcode() == BinaryOperator::Add) ? void (0) : __assert_fail
("cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4663, __extension__
__PRETTY_FUNCTION__))
;
4664 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar)(static_cast <bool> (cast<BinaryOperator>(NextIndVar
)->getOperand(0) == IndVar) ? void (0) : __assert_fail ("cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4664, __extension__
__PRETTY_FUNCTION__))
;
4665 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4666, __extension__
__PRETTY_FUNCTION__))
4666 ->isOne())(static_cast <bool> (cast<ConstantInt>(cast<BinaryOperator
>(NextIndVar)->getOperand(1)) ->isOne()) ? void (0) :
__assert_fail ("cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1)) ->isOne()"
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4666, __extension__
__PRETTY_FUNCTION__))
;
4667
4668 Value *TripCount = getTripCount();
4669 assert(TripCount && "Loop trip count not found?")(static_cast <bool> (TripCount && "Loop trip count not found?"
) ? void (0) : __assert_fail ("TripCount && \"Loop trip count not found?\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4669, __extension__
__PRETTY_FUNCTION__))
;
4670 assert(IndVar->getType() == TripCount->getType() &&(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4671, __extension__
__PRETTY_FUNCTION__))
4671 "Trip count and induction variable must have the same type")(static_cast <bool> (IndVar->getType() == TripCount->
getType() && "Trip count and induction variable must have the same type"
) ? void (0) : __assert_fail ("IndVar->getType() == TripCount->getType() && \"Trip count and induction variable must have the same type\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4671, __extension__
__PRETTY_FUNCTION__))
;
4672
4673 auto *CmpI = cast<CmpInst>(&Cond->front());
4674 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4675, __extension__
__PRETTY_FUNCTION__))
4675 "Exit condition must be a signed less-than comparison")(static_cast <bool> (CmpI->getPredicate() == CmpInst
::ICMP_ULT && "Exit condition must be a signed less-than comparison"
) ? void (0) : __assert_fail ("CmpI->getPredicate() == CmpInst::ICMP_ULT && \"Exit condition must be a signed less-than comparison\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4675, __extension__
__PRETTY_FUNCTION__))
;
4676 assert(CmpI->getOperand(0) == IndVar &&(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4677, __extension__
__PRETTY_FUNCTION__))
4677 "Exit condition must compare the induction variable")(static_cast <bool> (CmpI->getOperand(0) == IndVar &&
"Exit condition must compare the induction variable") ? void
(0) : __assert_fail ("CmpI->getOperand(0) == IndVar && \"Exit condition must compare the induction variable\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4677, __extension__
__PRETTY_FUNCTION__))
;
4678 assert(CmpI->getOperand(1) == TripCount &&(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4679, __extension__
__PRETTY_FUNCTION__))
4679 "Exit condition must compare with the trip count")(static_cast <bool> (CmpI->getOperand(1) == TripCount
&& "Exit condition must compare with the trip count"
) ? void (0) : __assert_fail ("CmpI->getOperand(1) == TripCount && \"Exit condition must compare with the trip count\""
, "llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp", 4679, __extension__
__PRETTY_FUNCTION__))
;
4680#endif
4681}
4682
4683void CanonicalLoopInfo::invalidate() {
4684 Header = nullptr;
4685 Cond = nullptr;
4686 Latch = nullptr;
4687 Exit = nullptr;
4688}

/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include/llvm/IR/Type.h

1//===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the Type class. For more "Type"
10// stuff, look in DerivedTypes.h.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_TYPE_H
15#define LLVM_IR_TYPE_H
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/Support/CBindingWrapping.h"
19#include "llvm/Support/Casting.h"
20#include "llvm/Support/Compiler.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Support/TypeSize.h"
23#include <cassert>
24#include <cstdint>
25#include <iterator>
26
27namespace llvm {
28
29class IntegerType;
30struct fltSemantics;
31class LLVMContext;
32class PointerType;
33class raw_ostream;
34class StringRef;
35template <typename PtrType> class SmallPtrSetImpl;
36
37/// The instances of the Type class are immutable: once they are created,
38/// they are never changed. Also note that only one instance of a particular
39/// type is ever created. Thus seeing if two types are equal is a matter of
40/// doing a trivial pointer comparison. To enforce that no two equal instances
41/// are created, Type instances can only be created via static factory methods
42/// in class Type and in derived classes. Once allocated, Types are never
43/// free'd.
44///
45class Type {
46public:
47 //===--------------------------------------------------------------------===//
48 /// Definitions of all of the base types for the Type system. Based on this
49 /// value, you can cast to a class defined in DerivedTypes.h.
50 /// Note: If you add an element to this, you need to add an element to the
51 /// Type::getPrimitiveType function, or else things will break!
52 /// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding.
53 ///
54 enum TypeID {
55 // PrimitiveTypes
56 HalfTyID = 0, ///< 16-bit floating point type
57 BFloatTyID, ///< 16-bit floating point type (7-bit significand)
58 FloatTyID, ///< 32-bit floating point type
59 DoubleTyID, ///< 64-bit floating point type
60 X86_FP80TyID, ///< 80-bit floating point type (X87)
61 FP128TyID, ///< 128-bit floating point type (112-bit significand)
62 PPC_FP128TyID, ///< 128-bit floating point type (two 64-bits, PowerPC)
63 VoidTyID, ///< type with no size
64 LabelTyID, ///< Labels
65 MetadataTyID, ///< Metadata
66 X86_MMXTyID, ///< MMX vectors (64 bits, X86 specific)
67 X86_AMXTyID, ///< AMX vectors (8192 bits, X86 specific)
68 TokenTyID, ///< Tokens
69
70 // Derived types... see DerivedTypes.h file.
71 IntegerTyID, ///< Arbitrary bit width integers
72 FunctionTyID, ///< Functions
73 PointerTyID, ///< Pointers
74 StructTyID, ///< Structures
75 ArrayTyID, ///< Arrays
76 FixedVectorTyID, ///< Fixed width SIMD vector type
77 ScalableVectorTyID, ///< Scalable SIMD vector type
78 TypedPointerTyID, ///< Typed pointer used by some GPU targets
79 };
80
81private:
82 /// This refers to the LLVMContext in which this type was uniqued.
83 LLVMContext &Context;
84
85 TypeID ID : 8; // The current base type of this type.
86 unsigned SubclassData : 24; // Space for subclasses to store data.
87 // Note that this should be synchronized with
88 // MAX_INT_BITS value in IntegerType class.
89
90protected:
91 friend class LLVMContextImpl;
92
93 explicit Type(LLVMContext &C, TypeID tid)
94 : Context(C), ID(tid), SubclassData(0) {}
95 ~Type() = default;
96
97 unsigned getSubclassData() const { return SubclassData; }
98
99 void setSubclassData(unsigned val) {
100 SubclassData = val;
101 // Ensure we don't have any accidental truncation.
102 assert(getSubclassData() == val && "Subclass data too large for field")(static_cast <bool> (getSubclassData() == val &&
"Subclass data too large for field") ? void (0) : __assert_fail
("getSubclassData() == val && \"Subclass data too large for field\""
, "llvm/include/llvm/IR/Type.h", 102, __extension__ __PRETTY_FUNCTION__
))
;
103 }
104
105 /// Keeps track of how many Type*'s there are in the ContainedTys list.
106 unsigned NumContainedTys = 0;
107
108 /// A pointer to the array of Types contained by this Type. For example, this
109 /// includes the arguments of a function type, the elements of a structure,
110 /// the pointee of a pointer, the element type of an array, etc. This pointer
111 /// may be 0 for types that don't contain other types (Integer, Double,
112 /// Float).
113 Type * const *ContainedTys = nullptr;
114
115public:
116 /// Print the current type.
117 /// Omit the type details if \p NoDetails == true.
118 /// E.g., let %st = type { i32, i16 }
119 /// When \p NoDetails is true, we only print %st.
120 /// Put differently, \p NoDetails prints the type as if
121 /// inlined with the operands when printing an instruction.
122 void print(raw_ostream &O, bool IsForDebug = false,
123 bool NoDetails = false) const;
124
125 void dump() const;
126
127 /// Return the LLVMContext in which this type was uniqued.
128 LLVMContext &getContext() const { return Context; }
129
130 //===--------------------------------------------------------------------===//
131 // Accessors for working with types.
132 //
133
134 /// Return the type id for the type. This will return one of the TypeID enum
135 /// elements defined above.
136 TypeID getTypeID() const { return ID; }
137
138 /// Return true if this is 'void'.
139 bool isVoidTy() const { return getTypeID() == VoidTyID; }
140
141 /// Return true if this is 'half', a 16-bit IEEE fp type.
142 bool isHalfTy() const { return getTypeID() == HalfTyID; }
143
144 /// Return true if this is 'bfloat', a 16-bit bfloat type.
145 bool isBFloatTy() const { return getTypeID() == BFloatTyID; }
146
147 /// Return true if this is a 16-bit float type.
148 bool is16bitFPTy() const {
149 return getTypeID() == BFloatTyID || getTypeID() == HalfTyID;
150 }
151
152 /// Return true if this is 'float', a 32-bit IEEE fp type.
153 bool isFloatTy() const { return getTypeID() == FloatTyID; }
154
155 /// Return true if this is 'double', a 64-bit IEEE fp type.
156 bool isDoubleTy() const { return getTypeID() == DoubleTyID; }
157
158 /// Return true if this is x86 long double.
159 bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; }
160
161 /// Return true if this is 'fp128'.
162 bool isFP128Ty() const { return getTypeID() == FP128TyID; }
163
164 /// Return true if this is powerpc long double.
165 bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; }
166
167 /// Return true if this is one of the six floating-point types
168 bool isFloatingPointTy() const {
169 return getTypeID() == HalfTyID || getTypeID() == BFloatTyID ||
170 getTypeID() == FloatTyID || getTypeID() == DoubleTyID ||
171 getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID ||
172 getTypeID() == PPC_FP128TyID;
173 }
174
175 const fltSemantics &getFltSemantics() const;
176
177 /// Return true if this is X86 MMX.
178 bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; }
179
180 /// Return true if this is X86 AMX.
181 bool isX86_AMXTy() const { return getTypeID() == X86_AMXTyID; }
182
183 /// Return true if this is a FP type or a vector of FP.
184 bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }
185
186 /// Return true if this is 'label'.
187 bool isLabelTy() const { return getTypeID() == LabelTyID; }
188
189 /// Return true if this is 'metadata'.
190 bool isMetadataTy() const { return getTypeID() == MetadataTyID; }
191
192 /// Return true if this is 'token'.
193 bool isTokenTy() const { return getTypeID() == TokenTyID; }
194
195 /// True if this is an instance of IntegerType.
196 bool isIntegerTy() const { return getTypeID() == IntegerTyID; }
8
Assuming the condition is false
9
Returning zero, which participates in a condition later
197
198 /// Return true if this is an IntegerType of the given width.
199 bool isIntegerTy(unsigned Bitwidth) const;
200
201 /// Return true if this is an integer type or a vector of integer types.
202 bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }
203
204 /// Return true if this is an integer type or a vector of integer types of
205 /// the given width.
206 bool isIntOrIntVectorTy(unsigned BitWidth) const {
207 return getScalarType()->isIntegerTy(BitWidth);
208 }
209
210 /// Return true if this is an integer type or a pointer type.
211 bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); }
212
213 /// True if this is an instance of FunctionType.
214 bool isFunctionTy() const { return getTypeID() == FunctionTyID; }
215
216 /// True if this is an instance of StructType.
217 bool isStructTy() const { return getTypeID() == StructTyID; }
218
219 /// True if this is an instance of ArrayType.
220 bool isArrayTy() const { return getTypeID() == ArrayTyID; }
221
222 /// True if this is an instance of PointerType.
223 bool isPointerTy() const { return getTypeID() == PointerTyID; }
224
225 /// True if this is an instance of an opaque PointerType.
226 bool isOpaquePointerTy() const;
227
228 /// Return true if this is a pointer type or a vector of pointer types.
229 bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); }
230
231 /// True if this is an instance of VectorType.
232 inline bool isVectorTy() const {
233 return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID;
234 }
235
236 /// Return true if this type could be converted with a lossless BitCast to
237 /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
238 /// same size only where no re-interpretation of the bits is done.
239 /// Determine if this type could be losslessly bitcast to Ty
240 bool canLosslesslyBitCastTo(Type *Ty) const;
241
242 /// Return true if this type is empty, that is, it has no elements or all of
243 /// its elements are empty.
244 bool isEmptyTy() const;
245
246 /// Return true if the type is "first class", meaning it is a valid type for a
247 /// Value.
248 bool isFirstClassType() const {
249 return getTypeID() != FunctionTyID && getTypeID() != VoidTyID;
250 }
251
252 /// Return true if the type is a valid type for a register in codegen. This
253 /// includes all first-class types except struct and array types.
254 bool isSingleValueType() const {
255 return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() ||
256 isPointerTy() || isVectorTy() || isX86_AMXTy();
257 }
258
259 /// Return true if the type is an aggregate type. This means it is valid as
260 /// the first operand of an insertvalue or extractvalue instruction. This
261 /// includes struct and array types, but does not include vector types.
262 bool isAggregateType() const {
263 return getTypeID() == StructTyID || getTypeID() == ArrayTyID;
264 }
265
266 /// Return true if it makes sense to take the size of this type. To get the
267 /// actual size for a particular target, it is reasonable to use the
268 /// DataLayout subsystem to do this.
269 bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const {
270 // If it's a primitive, it is always sized.
271 if (getTypeID() == IntegerTyID || isFloatingPointTy() ||
272 getTypeID() == PointerTyID || getTypeID() == X86_MMXTyID ||
273 getTypeID() == X86_AMXTyID)
274 return true;
275 // If it is not something that can have a size (e.g. a function or label),
276 // it doesn't have a size.
277 if (getTypeID() != StructTyID && getTypeID() != ArrayTyID && !isVectorTy())
278 return false;
279 // Otherwise we have to try harder to decide.
280 return isSizedDerivedType(Visited);
281 }
282
283 /// Return the basic size of this type if it is a primitive type. These are
284 /// fixed by LLVM and are not target-dependent.
285 /// This will return zero if the type does not have a size or is not a
286 /// primitive type.
287 ///
288 /// If this is a scalable vector type, the scalable property will be set and
289 /// the runtime size will be a positive integer multiple of the base size.
290 ///
291 /// Note that this may not reflect the size of memory allocated for an
292 /// instance of the type or the number of bytes that are written when an
293 /// instance of the type is stored to memory. The DataLayout class provides
294 /// additional query functions to provide this information.
295 ///
296 TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__));
297
298 /// If this is a vector type, return the getPrimitiveSizeInBits value for the
299 /// element type. Otherwise return the getPrimitiveSizeInBits value for this
300 /// type.
301 unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__));
302
303 /// Return the width of the mantissa of this type. This is only valid on
304 /// floating-point types. If the FP type does not have a stable mantissa (e.g.
305 /// ppc long double), this method returns -1.
306 int getFPMantissaWidth() const;
307
308 /// Return whether the type is IEEE compatible, as defined by the eponymous
309 /// method in APFloat.
310 bool isIEEE() const;
311
312 /// If this is a vector type, return the element type, otherwise return
313 /// 'this'.
314 inline Type *getScalarType() const {
315 if (isVectorTy())
316 return getContainedType(0);
317 return const_cast<Type *>(this);
318 }
319
320 //===--------------------------------------------------------------------===//
321 // Type Iteration support.
322 //
323 using subtype_iterator = Type * const *;
324
325 subtype_iterator subtype_begin() const { return ContainedTys; }
326 subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];}
327 ArrayRef<Type*> subtypes() const {
328 return makeArrayRef(subtype_begin(), subtype_end());
329 }
330
331 using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>;
332
333 subtype_reverse_iterator subtype_rbegin() const {
334 return subtype_reverse_iterator(subtype_end());
335 }
336 subtype_reverse_iterator subtype_rend() const {
337 return subtype_reverse_iterator(subtype_begin());
338 }
339
340 /// This method is used to implement the type iterator (defined at the end of
341 /// the file). For derived types, this returns the types 'contained' in the
342 /// derived type.
343 Type *getContainedType(unsigned i) const {
344 assert(i < NumContainedTys && "Index out of range!")(static_cast <bool> (i < NumContainedTys && "Index out of range!"
) ? void (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\""
, "llvm/include/llvm/IR/Type.h", 344, __extension__ __PRETTY_FUNCTION__
))
;
345 return ContainedTys[i];
346 }
347
348 /// Return the number of types in the derived type.
349 unsigned getNumContainedTypes() const { return NumContainedTys; }
350
351 //===--------------------------------------------------------------------===//
352 // Helper methods corresponding to subclass methods. This forces a cast to
353 // the specified subclass and calls its accessor. "getArrayNumElements" (for
354 // example) is shorthand for cast<ArrayType>(Ty)->getNumElements(). This is
355 // only intended to cover the core methods that are frequently used, helper
356 // methods should not be added here.
357
358 inline unsigned getIntegerBitWidth() const;
359
360 inline Type *getFunctionParamType(unsigned i) const;
361 inline unsigned getFunctionNumParams() const;
362 inline bool isFunctionVarArg() const;
363
364 inline StringRef getStructName() const;
365 inline unsigned getStructNumElements() const;
366 inline Type *getStructElementType(unsigned N) const;
367
368 inline uint64_t getArrayNumElements() const;
369
370 Type *getArrayElementType() const {
371 assert(getTypeID() == ArrayTyID)(static_cast <bool> (getTypeID() == ArrayTyID) ? void (
0) : __assert_fail ("getTypeID() == ArrayTyID", "llvm/include/llvm/IR/Type.h"
, 371, __extension__ __PRETTY_FUNCTION__))
;
372 return ContainedTys[0];
373 }
374
375 /// This method is deprecated without replacement. Pointer element types are
376 /// not available with opaque pointers.
377 [[deprecated("Deprecated without replacement, see "
378 "https://llvm.org/docs/OpaquePointers.html for context and "
379 "migration instructions")]]
380 Type *getPointerElementType() const {
381 return getNonOpaquePointerElementType();
382 }
383
384 /// Only use this method in code that is not reachable with opaque pointers,
385 /// or part of deprecated methods that will be removed as part of the opaque
386 /// pointers transition.
387 Type *getNonOpaquePointerElementType() const {
388 assert(getTypeID() == PointerTyID)(static_cast <bool> (getTypeID() == PointerTyID) ? void
(0) : __assert_fail ("getTypeID() == PointerTyID", "llvm/include/llvm/IR/Type.h"
, 388, __extension__ __PRETTY_FUNCTION__))
;
389 assert(NumContainedTys &&(static_cast <bool> (NumContainedTys && "Attempting to get element type of opaque pointer"
) ? void (0) : __assert_fail ("NumContainedTys && \"Attempting to get element type of opaque pointer\""
, "llvm/include/llvm/IR/Type.h", 390, __extension__ __PRETTY_FUNCTION__
))
390 "Attempting to get element type of opaque pointer")(static_cast <bool> (NumContainedTys && "Attempting to get element type of opaque pointer"
) ? void (0) : __assert_fail ("NumContainedTys && \"Attempting to get element type of opaque pointer\""
, "llvm/include/llvm/IR/Type.h", 390, __extension__ __PRETTY_FUNCTION__
))
;
391 return ContainedTys[0];
392 }
393
394 /// Given vector type, change the element type,
395 /// whilst keeping the old number of elements.
396 /// For non-vectors simply returns \p EltTy.
397 inline Type *getWithNewType(Type *EltTy) const;
398
399 /// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
400 /// whilst keeping the old number of lanes.
401 inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
402
403 /// Given scalar/vector integer type, returns a type with elements twice as
404 /// wide as in the original type. For vectors, preserves element count.
405 inline Type *getExtendedType() const;
406
407 /// Get the address space of this pointer or pointer vector type.
408 inline unsigned getPointerAddressSpace() const;
409
410 //===--------------------------------------------------------------------===//
411 // Static members exported by the Type class itself. Useful for getting
412 // instances of Type.
413 //
414
415 /// Return a type based on an identifier.
416 static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);
417
418 //===--------------------------------------------------------------------===//
419 // These are the builtin types that are always available.
420 //
421 static Type *getVoidTy(LLVMContext &C);
422 static Type *getLabelTy(LLVMContext &C);
423 static Type *getHalfTy(LLVMContext &C);
424 static Type *getBFloatTy(LLVMContext &C);
425 static Type *getFloatTy(LLVMContext &C);
426 static Type *getDoubleTy(LLVMContext &C);
427 static Type *getMetadataTy(LLVMContext &C);
428 static Type *getX86_FP80Ty(LLVMContext &C);
429 static Type *getFP128Ty(LLVMContext &C);
430 static Type *getPPC_FP128Ty(LLVMContext &C);
431 static Type *getX86_MMXTy(LLVMContext &C);
432 static Type *getX86_AMXTy(LLVMContext &C);
433 static Type *getTokenTy(LLVMContext &C);
434 static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
435 static IntegerType *getInt1Ty(LLVMContext &C);
436 static IntegerType *getInt8Ty(LLVMContext &C);
437 static IntegerType *getInt16Ty(LLVMContext &C);
438 static IntegerType *getInt32Ty(LLVMContext &C);
439 static IntegerType *getInt64Ty(LLVMContext &C);
440 static IntegerType *getInt128Ty(LLVMContext &C);
441 template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
442 int noOfBits = sizeof(ScalarTy) * CHAR_BIT8;
443 if (std::is_integral<ScalarTy>::value) {
444 return (Type*) Type::getIntNTy(C, noOfBits);
445 } else if (std::is_floating_point<ScalarTy>::value) {
446 switch (noOfBits) {
447 case 32:
448 return Type::getFloatTy(C);
449 case 64:
450 return Type::getDoubleTy(C);
451 }
452 }
453 llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy"
, "llvm/include/llvm/IR/Type.h", 453)
;
454 }
455 static Type *getFloatingPointTy(LLVMContext &C, const fltSemantics &S);
456
457 //===--------------------------------------------------------------------===//
458 // Convenience methods for getting pointer types with one of the above builtin
459 // types as pointee.
460 //
461 static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0);
462 static PointerType *getBFloatPtrTy(LLVMContext &C, unsigned AS = 0);
463 static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0);
464 static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0);
465 static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0);
466 static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0);
467 static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0);
468 static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0);
469 static PointerType *getX86_AMXPtrTy(LLVMContext &C, unsigned AS = 0);
470 static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0);
471 static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0);
472 static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0);
473 static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0);
474 static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0);
475 static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0);
476
477 /// Return a pointer to the current type. This is equivalent to
478 /// PointerType::get(Foo, AddrSpace).
479 /// TODO: Remove this after opaque pointer transition is complete.
480 PointerType *getPointerTo(unsigned AddrSpace = 0) const;
481
482private:
483 /// Derived types like structures and arrays are sized iff all of the members
484 /// of the type are sized as well. Since asking for their size is relatively
485 /// uncommon, move this operation out-of-line.
486 bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
487};
488
489// Printing of types.
490inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
491 T.print(OS);
492 return OS;
493}
494
495// allow isa<PointerType>(x) to work without DerivedTypes.h included.
496template <> struct isa_impl<PointerType, Type> {
497 static inline bool doit(const Type &Ty) {
498 return Ty.getTypeID() == Type::PointerTyID;
499 }
500};
501
502// Create wrappers for C Binding types (see CBindingWrapping.h).
503DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast<
Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return
reinterpret_cast<LLVMTypeRef>(const_cast<Type*>(
P)); } template<typename T> inline T *unwrap(LLVMTypeRef
P) { return cast<T>(unwrap(P)); }
504
505/* Specialized opaque type conversions.
506 */
507inline Type **unwrap(LLVMTypeRef* Tys) {
508 return reinterpret_cast<Type**>(Tys);
509}
510
511inline LLVMTypeRef *wrap(Type **Tys) {
512 return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
513}
514
515} // end namespace llvm
516
517#endif // LLVM_IR_TYPE_H

/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugLoc.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/FPEnv.h"