Bug Summary

File:tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8488, column 15
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn374877/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn374877/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn374877=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-15-233810-7101-1 -x c++ /build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGOpenMPRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/CodeGen/ConstantInitBuilder.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/StmtOpenMP.h"
21#include "clang/Basic/BitmaskEnum.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30
31using namespace clang;
32using namespace CodeGen;
33
34namespace {
35/// Base class for handling code generation inside OpenMP regions.
36class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
37public:
38 /// Kinds of OpenMP regions used in codegen.
39 enum CGOpenMPRegionKind {
40 /// Region with outlined function for standalone 'parallel'
41 /// directive.
42 ParallelOutlinedRegion,
43 /// Region with outlined function for standalone 'task' directive.
44 TaskOutlinedRegion,
45 /// Region for constructs that do not require function outlining,
46 /// like 'for', 'sections', 'atomic' etc. directives.
47 InlinedRegion,
48 /// Region with outlined function for standalone 'target' directive.
49 TargetRegion,
50 };
51
52 CGOpenMPRegionInfo(const CapturedStmt &CS,
53 const CGOpenMPRegionKind RegionKind,
54 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
55 bool HasCancel)
56 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58
59 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
60 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61 bool HasCancel)
62 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
64
65 /// Get a variable or parameter for storing global thread id
66 /// inside OpenMP construct.
67 virtual const VarDecl *getThreadIDVariable() const = 0;
68
69 /// Emit the captured statement body.
70 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
71
72 /// Get an LValue for the current ThreadID variable.
73 /// \return LValue for thread id variable. This LValue always has type int32*.
74 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75
76 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
77
78 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
79
80 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
81
82 bool hasCancel() const { return HasCancel; }
83
84 static bool classof(const CGCapturedStmtInfo *Info) {
85 return Info->getKind() == CR_OpenMP;
86 }
87
88 ~CGOpenMPRegionInfo() override = default;
89
90protected:
91 CGOpenMPRegionKind RegionKind;
92 RegionCodeGenTy CodeGen;
93 OpenMPDirectiveKind Kind;
94 bool HasCancel;
95};
96
97/// API for captured statement code generation in OpenMP constructs.
98class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
99public:
100 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
101 const RegionCodeGenTy &CodeGen,
102 OpenMPDirectiveKind Kind, bool HasCancel,
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
105 HasCancel),
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 107, __PRETTY_FUNCTION__))
;
108 }
109
110 /// Get a variable or parameter for storing global thread id
111 /// inside OpenMP construct.
112 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
113
114 /// Get the name of the capture helper.
115 StringRef getHelperName() const override { return HelperName; }
116
117 static bool classof(const CGCapturedStmtInfo *Info) {
118 return CGOpenMPRegionInfo::classof(Info) &&
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
121 }
122
123private:
124 /// A variable or parameter storing global thread id for OpenMP
125 /// constructs.
126 const VarDecl *ThreadIDVar;
127 StringRef HelperName;
128};
129
130/// API for captured statement code generation in OpenMP constructs.
131class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
132public:
133 class UntiedTaskActionTy final : public PrePostActionTy {
134 bool Untied;
135 const VarDecl *PartIDVar;
136 const RegionCodeGenTy UntiedCodeGen;
137 llvm::SwitchInst *UntiedSwitch = nullptr;
138
139 public:
140 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
141 const RegionCodeGenTy &UntiedCodeGen)
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
143 void Enter(CodeGenFunction &CGF) override {
144 if (Untied) {
145 // Emit task switching point.
146 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
147 CGF.GetAddrOfLocalVar(PartIDVar),
148 PartIDVar->getType()->castAs<PointerType>());
149 llvm::Value *Res =
150 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
151 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
152 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153 CGF.EmitBlock(DoneBB);
154 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157 CGF.Builder.GetInsertBlock());
158 emitUntiedSwitch(CGF);
159 }
160 }
161 void emitUntiedSwitch(CodeGenFunction &CGF) const {
162 if (Untied) {
163 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
164 CGF.GetAddrOfLocalVar(PartIDVar),
165 PartIDVar->getType()->castAs<PointerType>());
166 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167 PartIdLVal);
168 UntiedCodeGen(CGF);
169 CodeGenFunction::JumpDest CurPoint =
170 CGF.getJumpDestInCurrentScope(".untied.next.");
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174 CGF.Builder.GetInsertBlock());
175 CGF.EmitBranchThroughCleanup(CurPoint);
176 CGF.EmitBlock(CurPoint.getBlock());
177 }
178 }
179 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180 };
181 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
182 const VarDecl *ThreadIDVar,
183 const RegionCodeGenTy &CodeGen,
184 OpenMPDirectiveKind Kind, bool HasCancel,
185 const UntiedTaskActionTy &Action)
186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187 ThreadIDVar(ThreadIDVar), Action(Action) {
188 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 188, __PRETTY_FUNCTION__))
;
189 }
190
191 /// Get a variable or parameter for storing global thread id
192 /// inside OpenMP construct.
193 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
194
195 /// Get an LValue for the current ThreadID variable.
196 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197
198 /// Get the name of the capture helper.
199 StringRef getHelperName() const override { return ".omp_outlined."; }
200
201 void emitUntiedSwitch(CodeGenFunction &CGF) override {
202 Action.emitUntiedSwitch(CGF);
203 }
204
205 static bool classof(const CGCapturedStmtInfo *Info) {
206 return CGOpenMPRegionInfo::classof(Info) &&
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208 TaskOutlinedRegion;
209 }
210
211private:
212 /// A variable or parameter storing global thread id for OpenMP
213 /// constructs.
214 const VarDecl *ThreadIDVar;
215 /// Action for emitting code for untied tasks.
216 const UntiedTaskActionTy &Action;
217};
218
219/// API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
223 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
224 const RegionCodeGenTy &CodeGen,
225 OpenMPDirectiveKind Kind, bool HasCancel)
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OldCSI(OldCSI),
228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229
230 // Retrieve the value of the context parameter.
231 llvm::Value *getContextValue() const override {
232 if (OuterRegionInfo)
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 234)
;
235 }
236
237 void setContextValue(llvm::Value *V) override {
238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
240 return;
241 }
242 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 242)
;
243 }
244
245 /// Lookup the captured field decl for a variable.
246 const FieldDecl *lookup(const VarDecl *VD) const override {
247 if (OuterRegionInfo)
248 return OuterRegionInfo->lookup(VD);
249 // If there is no outer outlined region,no need to lookup in a list of
250 // captured variables, we can use the original one.
251 return nullptr;
252 }
253
254 FieldDecl *getThisFieldDecl() const override {
255 if (OuterRegionInfo)
256 return OuterRegionInfo->getThisFieldDecl();
257 return nullptr;
258 }
259
260 /// Get a variable or parameter for storing global thread id
261 /// inside OpenMP construct.
262 const VarDecl *getThreadIDVariable() const override {
263 if (OuterRegionInfo)
264 return OuterRegionInfo->getThreadIDVariable();
265 return nullptr;
266 }
267
268 /// Get an LValue for the current ThreadID variable.
269 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270 if (OuterRegionInfo)
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 272)
;
273 }
274
275 /// Get the name of the capture helper.
276 StringRef getHelperName() const override {
277 if (auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
279 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 279)
;
280 }
281
282 void emitUntiedSwitch(CodeGenFunction &CGF) override {
283 if (OuterRegionInfo)
284 OuterRegionInfo->emitUntiedSwitch(CGF);
285 }
286
287 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
289 static bool classof(const CGCapturedStmtInfo *Info) {
290 return CGOpenMPRegionInfo::classof(Info) &&
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292 }
293
294 ~CGOpenMPInlinedRegionInfo() override = default;
295
296private:
297 /// CodeGen info about outer OpenMP region.
298 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299 CGOpenMPRegionInfo *OuterRegionInfo;
300};
301
302/// API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
308public:
309 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
310 const RegionCodeGenTy &CodeGen, StringRef HelperName)
311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
312 /*HasCancel=*/false),
313 HelperName(HelperName) {}
314
315 /// This is unused for target regions because each starts executing
316 /// with a single thread.
317 const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319 /// Get the name of the capture helper.
320 StringRef getHelperName() const override { return HelperName; }
321
322 static bool classof(const CGCapturedStmtInfo *Info) {
323 return CGOpenMPRegionInfo::classof(Info) &&
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325 }
326
327private:
328 StringRef HelperName;
329};
330
331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
332 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 332)
;
333}
334/// API for generation of expressions captured in a innermost OpenMP
335/// region.
336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
337public:
338 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340 OMPD_unknown,
341 /*HasCancel=*/false),
342 PrivScope(CGF) {
343 // Make sure the globals captured in the provided statement are local by
344 // using the privatization logic. We assume the same variable is not
345 // captured more than once.
346 for (const auto &C : CS.captures()) {
347 if (!C.capturesVariable() && !C.capturesVariableByCopy())
348 continue;
349
350 const VarDecl *VD = C.getCapturedVar();
351 if (VD->isLocalVarDeclOrParm())
352 continue;
353
354 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
355 /*RefersToEnclosingVariableOrCapture=*/false,
356 VD->getType().getNonReferenceType(), VK_LValue,
357 C.getLocation());
358 PrivScope.addPrivate(
359 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
360 }
361 (void)PrivScope.Privatize();
362 }
363
364 /// Lookup the captured field decl for a variable.
365 const FieldDecl *lookup(const VarDecl *VD) const override {
366 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
367 return FD;
368 return nullptr;
369 }
370
371 /// Emit the captured statement body.
372 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
373 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 373)
;
374 }
375
376 /// Get a variable or parameter for storing global thread id
377 /// inside OpenMP construct.
378 const VarDecl *getThreadIDVariable() const override {
379 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 379)
;
380 }
381
382 /// Get the name of the capture helper.
383 StringRef getHelperName() const override {
384 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 384)
;
385 }
386
387 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
388
389private:
390 /// Private scope to capture global variables.
391 CodeGenFunction::OMPPrivateScope PrivScope;
392};
393
394/// RAII for emitting code of OpenMP constructs.
395class InlinedOpenMPRegionRAII {
396 CodeGenFunction &CGF;
397 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398 FieldDecl *LambdaThisCaptureField = nullptr;
399 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
400
401public:
402 /// Constructs region for combined constructs.
403 /// \param CodeGen Code generation sequence for combined directives. Includes
404 /// a list of functions used for code generation of implicitly inlined
405 /// regions.
406 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
407 OpenMPDirectiveKind Kind, bool HasCancel)
408 : CGF(CGF) {
409 // Start emission for the construct.
410 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414 CGF.LambdaThisCaptureField = nullptr;
415 BlockInfo = CGF.BlockInfo;
416 CGF.BlockInfo = nullptr;
417 }
418
419 ~InlinedOpenMPRegionRAII() {
420 // Restore original CapturedStmtInfo only if we're done with code emission.
421 auto *OldCSI =
422 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
423 delete CGF.CapturedStmtInfo;
424 CGF.CapturedStmtInfo = OldCSI;
425 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
426 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
427 CGF.BlockInfo = BlockInfo;
428 }
429};
430
431/// Values for bit flags used in the ident_t to describe the fields.
432/// All enumeric elements are named and described in accordance with the code
433/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
434enum OpenMPLocationFlags : unsigned {
435 /// Use trampoline for internal microtask.
436 OMP_IDENT_IMD = 0x01,
437 /// Use c-style ident structure.
438 OMP_IDENT_KMPC = 0x02,
439 /// Atomic reduction option for kmpc_reduce.
440 OMP_ATOMIC_REDUCE = 0x10,
441 /// Explicit 'barrier' directive.
442 OMP_IDENT_BARRIER_EXPL = 0x20,
443 /// Implicit barrier in code.
444 OMP_IDENT_BARRIER_IMPL = 0x40,
445 /// Implicit barrier in 'for' directive.
446 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
447 /// Implicit barrier in 'sections' directive.
448 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
449 /// Implicit barrier in 'single' directive.
450 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
451 /// Call of __kmp_for_static_init for static loop.
452 OMP_IDENT_WORK_LOOP = 0x200,
453 /// Call of __kmp_for_static_init for sections.
454 OMP_IDENT_WORK_SECTIONS = 0x400,
455 /// Call of __kmp_for_static_init for distribute.
456 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
458};
459
460namespace {
461LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
462/// Values for bit flags for marking which requires clauses have been used.
463enum OpenMPOffloadingRequiresDirFlags : int64_t {
464 /// flag undefined.
465 OMP_REQ_UNDEFINED = 0x000,
466 /// no requires clause present.
467 OMP_REQ_NONE = 0x001,
468 /// reverse_offload clause.
469 OMP_REQ_REVERSE_OFFLOAD = 0x002,
470 /// unified_address clause.
471 OMP_REQ_UNIFIED_ADDRESS = 0x004,
472 /// unified_shared_memory clause.
473 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
474 /// dynamic_allocators clause.
475 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
476 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
477};
478
479enum OpenMPOffloadingReservedDeviceIDs {
480 /// Device ID if the device was not defined, runtime should get it
481 /// from environment variables in the spec.
482 OMP_DEVICEID_UNDEF = -1,
483};
484} // anonymous namespace
485
486/// Describes ident structure that describes a source location.
487/// All descriptions are taken from
488/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
489/// Original structure:
490/// typedef struct ident {
491/// kmp_int32 reserved_1; /**< might be used in Fortran;
492/// see above */
493/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
494/// KMP_IDENT_KMPC identifies this union
495/// member */
496/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
497/// see above */
498///#if USE_ITT_BUILD
499/// /* but currently used for storing
500/// region-specific ITT */
501/// /* contextual information. */
502///#endif /* USE_ITT_BUILD */
503/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
504/// C++ */
505/// char const *psource; /**< String describing the source location.
506/// The string is composed of semi-colon separated
507// fields which describe the source file,
508/// the function and a pair of line numbers that
509/// delimit the construct.
510/// */
511/// } ident_t;
512enum IdentFieldIndex {
513 /// might be used in Fortran
514 IdentField_Reserved_1,
515 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
516 IdentField_Flags,
517 /// Not really used in Fortran any more
518 IdentField_Reserved_2,
519 /// Source[4] in Fortran, do not use for C++
520 IdentField_Reserved_3,
521 /// String describing the source location. The string is composed of
522 /// semi-colon separated fields which describe the source file, the function
523 /// and a pair of line numbers that delimit the construct.
524 IdentField_PSource
525};
526
527/// Schedule types for 'omp for' loops (these enumerators are taken from
528/// the enum sched_type in kmp.h).
529enum OpenMPSchedType {
530 /// Lower bound for default (unordered) versions.
531 OMP_sch_lower = 32,
532 OMP_sch_static_chunked = 33,
533 OMP_sch_static = 34,
534 OMP_sch_dynamic_chunked = 35,
535 OMP_sch_guided_chunked = 36,
536 OMP_sch_runtime = 37,
537 OMP_sch_auto = 38,
538 /// static with chunk adjustment (e.g., simd)
539 OMP_sch_static_balanced_chunked = 45,
540 /// Lower bound for 'ordered' versions.
541 OMP_ord_lower = 64,
542 OMP_ord_static_chunked = 65,
543 OMP_ord_static = 66,
544 OMP_ord_dynamic_chunked = 67,
545 OMP_ord_guided_chunked = 68,
546 OMP_ord_runtime = 69,
547 OMP_ord_auto = 70,
548 OMP_sch_default = OMP_sch_static,
549 /// dist_schedule types
550 OMP_dist_sch_static_chunked = 91,
551 OMP_dist_sch_static = 92,
552 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
553 /// Set if the monotonic schedule modifier was present.
554 OMP_sch_modifier_monotonic = (1 << 29),
555 /// Set if the nonmonotonic schedule modifier was present.
556 OMP_sch_modifier_nonmonotonic = (1 << 30),
557};
558
559enum OpenMPRTLFunction {
560 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
561 /// kmpc_micro microtask, ...);
562 OMPRTL__kmpc_fork_call,
563 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
564 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
565 OMPRTL__kmpc_threadprivate_cached,
566 /// Call to void __kmpc_threadprivate_register( ident_t *,
567 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
568 OMPRTL__kmpc_threadprivate_register,
569 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
570 OMPRTL__kmpc_global_thread_num,
571 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
572 // kmp_critical_name *crit);
573 OMPRTL__kmpc_critical,
574 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
575 // global_tid, kmp_critical_name *crit, uintptr_t hint);
576 OMPRTL__kmpc_critical_with_hint,
577 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
578 // kmp_critical_name *crit);
579 OMPRTL__kmpc_end_critical,
580 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
581 // global_tid);
582 OMPRTL__kmpc_cancel_barrier,
583 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
584 OMPRTL__kmpc_barrier,
585 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
586 OMPRTL__kmpc_for_static_fini,
587 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
588 // global_tid);
589 OMPRTL__kmpc_serialized_parallel,
590 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
591 // global_tid);
592 OMPRTL__kmpc_end_serialized_parallel,
593 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
594 // kmp_int32 num_threads);
595 OMPRTL__kmpc_push_num_threads,
596 // Call to void __kmpc_flush(ident_t *loc);
597 OMPRTL__kmpc_flush,
598 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
599 OMPRTL__kmpc_master,
600 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
601 OMPRTL__kmpc_end_master,
602 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
603 // int end_part);
604 OMPRTL__kmpc_omp_taskyield,
605 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
606 OMPRTL__kmpc_single,
607 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
608 OMPRTL__kmpc_end_single,
609 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
610 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
611 // kmp_routine_entry_t *task_entry);
612 OMPRTL__kmpc_omp_task_alloc,
613 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
614 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
615 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
616 // kmp_int64 device_id);
617 OMPRTL__kmpc_omp_target_task_alloc,
618 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
619 // new_task);
620 OMPRTL__kmpc_omp_task,
621 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
622 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
623 // kmp_int32 didit);
624 OMPRTL__kmpc_copyprivate,
625 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
626 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
627 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
628 OMPRTL__kmpc_reduce,
629 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
630 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
631 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
632 // *lck);
633 OMPRTL__kmpc_reduce_nowait,
634 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
635 // kmp_critical_name *lck);
636 OMPRTL__kmpc_end_reduce,
637 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
638 // kmp_critical_name *lck);
639 OMPRTL__kmpc_end_reduce_nowait,
640 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
641 // kmp_task_t * new_task);
642 OMPRTL__kmpc_omp_task_begin_if0,
643 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
644 // kmp_task_t * new_task);
645 OMPRTL__kmpc_omp_task_complete_if0,
646 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
647 OMPRTL__kmpc_ordered,
648 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
649 OMPRTL__kmpc_end_ordered,
650 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
651 // global_tid);
652 OMPRTL__kmpc_omp_taskwait,
653 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
654 OMPRTL__kmpc_taskgroup,
655 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
656 OMPRTL__kmpc_end_taskgroup,
657 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
658 // int proc_bind);
659 OMPRTL__kmpc_push_proc_bind,
660 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
661 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
662 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
663 OMPRTL__kmpc_omp_task_with_deps,
664 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
665 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
666 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
667 OMPRTL__kmpc_omp_wait_deps,
668 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
669 // global_tid, kmp_int32 cncl_kind);
670 OMPRTL__kmpc_cancellationpoint,
671 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
672 // kmp_int32 cncl_kind);
673 OMPRTL__kmpc_cancel,
674 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
675 // kmp_int32 num_teams, kmp_int32 thread_limit);
676 OMPRTL__kmpc_push_num_teams,
677 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
678 // microtask, ...);
679 OMPRTL__kmpc_fork_teams,
680 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
681 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
682 // sched, kmp_uint64 grainsize, void *task_dup);
683 OMPRTL__kmpc_taskloop,
684 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
685 // num_dims, struct kmp_dim *dims);
686 OMPRTL__kmpc_doacross_init,
687 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
688 OMPRTL__kmpc_doacross_fini,
689 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
690 // *vec);
691 OMPRTL__kmpc_doacross_post,
692 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
693 // *vec);
694 OMPRTL__kmpc_doacross_wait,
695 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
696 // *data);
697 OMPRTL__kmpc_task_reduction_init,
698 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
699 // *d);
700 OMPRTL__kmpc_task_reduction_get_th_data,
701 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
702 OMPRTL__kmpc_alloc,
703 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
704 OMPRTL__kmpc_free,
705
706 //
707 // Offloading related calls
708 //
709 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
710 // size);
711 OMPRTL__kmpc_push_target_tripcount,
712 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
713 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
714 // *arg_types);
715 OMPRTL__tgt_target,
716 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
717 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
718 // *arg_types);
719 OMPRTL__tgt_target_nowait,
720 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
721 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
722 // *arg_types, int32_t num_teams, int32_t thread_limit);
723 OMPRTL__tgt_target_teams,
724 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
725 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
726 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
727 OMPRTL__tgt_target_teams_nowait,
728 // Call to void __tgt_register_requires(int64_t flags);
729 OMPRTL__tgt_register_requires,
730 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
731 OMPRTL__tgt_register_lib,
732 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
733 OMPRTL__tgt_unregister_lib,
734 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
735 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
736 OMPRTL__tgt_target_data_begin,
737 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
738 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
739 // *arg_types);
740 OMPRTL__tgt_target_data_begin_nowait,
741 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
742 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
743 OMPRTL__tgt_target_data_end,
744 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
745 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
746 // *arg_types);
747 OMPRTL__tgt_target_data_end_nowait,
748 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
749 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
750 OMPRTL__tgt_target_data_update,
751 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
752 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
753 // *arg_types);
754 OMPRTL__tgt_target_data_update_nowait,
755 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
756 OMPRTL__tgt_mapper_num_components,
757 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
758 // *base, void *begin, int64_t size, int64_t type);
759 OMPRTL__tgt_push_mapper_component,
760};
761
762/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
763/// region.
764class CleanupTy final : public EHScopeStack::Cleanup {
765 PrePostActionTy *Action;
766
767public:
768 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
769 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
770 if (!CGF.HaveInsertPoint())
771 return;
772 Action->Exit(CGF);
773 }
774};
775
776} // anonymous namespace
777
778void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
779 CodeGenFunction::RunCleanupsScope Scope(CGF);
780 if (PrePostAction
11.1
Field 'PrePostAction' is null
11.1
Field 'PrePostAction' is null
) {
12
Taking false branch
781 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
782 Callback(CodeGen, CGF, *PrePostAction);
783 } else {
784 PrePostActionTy Action;
785 Callback(CodeGen, CGF, Action);
13
Calling 'RegionCodeGenTy::CallbackFn'
786 }
787}
788
789/// Check if the combiner is a call to UDR combiner and if it is so return the
790/// UDR decl used for reduction.
791static const OMPDeclareReductionDecl *
792getReductionInit(const Expr *ReductionOp) {
793 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
794 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
795 if (const auto *DRE =
796 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
797 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
798 return DRD;
799 return nullptr;
800}
801
802static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
803 const OMPDeclareReductionDecl *DRD,
804 const Expr *InitOp,
805 Address Private, Address Original,
806 QualType Ty) {
807 if (DRD->getInitializer()) {
808 std::pair<llvm::Function *, llvm::Function *> Reduction =
809 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
810 const auto *CE = cast<CallExpr>(InitOp);
811 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
812 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
813 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
814 const auto *LHSDRE =
815 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
816 const auto *RHSDRE =
817 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
818 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
819 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
820 [=]() { return Private; });
821 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
822 [=]() { return Original; });
823 (void)PrivateScope.Privatize();
824 RValue Func = RValue::get(Reduction.second);
825 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
826 CGF.EmitIgnoredExpr(InitOp);
827 } else {
828 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
829 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
830 auto *GV = new llvm::GlobalVariable(
831 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
832 llvm::GlobalValue::PrivateLinkage, Init, Name);
833 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
834 RValue InitRVal;
835 switch (CGF.getEvaluationKind(Ty)) {
836 case TEK_Scalar:
837 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
838 break;
839 case TEK_Complex:
840 InitRVal =
841 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
842 break;
843 case TEK_Aggregate:
844 InitRVal = RValue::getAggregate(LV.getAddress());
845 break;
846 }
847 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
848 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
849 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
850 /*IsInitializer=*/false);
851 }
852}
853
854/// Emit initialization of arrays of complex types.
855/// \param DestAddr Address of the array.
856/// \param Type Type of array.
857/// \param Init Initial expression of array.
858/// \param SrcAddr Address of the original array.
859static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
860 QualType Type, bool EmitDeclareReductionInit,
861 const Expr *Init,
862 const OMPDeclareReductionDecl *DRD,
863 Address SrcAddr = Address::invalid()) {
864 // Perform element-by-element initialization.
865 QualType ElementTy;
866
867 // Drill down to the base element type on both arrays.
868 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
869 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
870 DestAddr =
871 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
872 if (DRD)
873 SrcAddr =
874 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
875
876 llvm::Value *SrcBegin = nullptr;
877 if (DRD)
878 SrcBegin = SrcAddr.getPointer();
879 llvm::Value *DestBegin = DestAddr.getPointer();
880 // Cast from pointer to array type to pointer to single element.
881 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
882 // The basic structure here is a while-do loop.
883 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
884 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
885 llvm::Value *IsEmpty =
886 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
887 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
888
889 // Enter the loop body, making that address the current address.
890 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
891 CGF.EmitBlock(BodyBB);
892
893 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
894
895 llvm::PHINode *SrcElementPHI = nullptr;
896 Address SrcElementCurrent = Address::invalid();
897 if (DRD) {
898 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
899 "omp.arraycpy.srcElementPast");
900 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
901 SrcElementCurrent =
902 Address(SrcElementPHI,
903 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
904 }
905 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
906 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
907 DestElementPHI->addIncoming(DestBegin, EntryBB);
908 Address DestElementCurrent =
909 Address(DestElementPHI,
910 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
911
912 // Emit copy.
913 {
914 CodeGenFunction::RunCleanupsScope InitScope(CGF);
915 if (EmitDeclareReductionInit) {
916 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
917 SrcElementCurrent, ElementTy);
918 } else
919 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
920 /*IsInitializer=*/false);
921 }
922
923 if (DRD) {
924 // Shift the address forward by one element.
925 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
926 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
927 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
928 }
929
930 // Shift the address forward by one element.
931 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
932 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
933 // Check whether we've reached the end.
934 llvm::Value *Done =
935 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
936 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
937 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
938
939 // Done.
940 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
941}
942
943LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
944 return CGF.EmitOMPSharedLValue(E);
945}
946
947LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
948 const Expr *E) {
949 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
950 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
951 return LValue();
952}
953
954void ReductionCodeGen::emitAggregateInitialization(
955 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
956 const OMPDeclareReductionDecl *DRD) {
957 // Emit VarDecl with copy init for arrays.
958 // Get the address of the original variable captured in current
959 // captured region.
960 const auto *PrivateVD =
961 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
962 bool EmitDeclareReductionInit =
963 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
964 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
965 EmitDeclareReductionInit,
966 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
967 : PrivateVD->getInit(),
968 DRD, SharedLVal.getAddress());
969}
970
971ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
972 ArrayRef<const Expr *> Privates,
973 ArrayRef<const Expr *> ReductionOps) {
974 ClausesData.reserve(Shareds.size());
975 SharedAddresses.reserve(Shareds.size());
976 Sizes.reserve(Shareds.size());
977 BaseDecls.reserve(Shareds.size());
978 auto IPriv = Privates.begin();
979 auto IRed = ReductionOps.begin();
980 for (const Expr *Ref : Shareds) {
981 ClausesData.emplace_back(Ref, *IPriv, *IRed);
982 std::advance(IPriv, 1);
983 std::advance(IRed, 1);
984 }
985}
986
987void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
988 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
989 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
;
990 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
991 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
992 SharedAddresses.emplace_back(First, Second);
993}
994
995void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
996 const auto *PrivateVD =
997 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
998 QualType PrivateType = PrivateVD->getType();
999 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1000 if (!PrivateType->isVariablyModifiedType()) {
1001 Sizes.emplace_back(
1002 CGF.getTypeSize(
1003 SharedAddresses[N].first.getType().getNonReferenceType()),
1004 nullptr);
1005 return;
1006 }
1007 llvm::Value *Size;
1008 llvm::Value *SizeInChars;
1009 auto *ElemType =
1010 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
1011 ->getElementType();
1012 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1013 if (AsArraySection) {
1014 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
1015 SharedAddresses[N].first.getPointer());
1016 Size = CGF.Builder.CreateNUWAdd(
1017 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1018 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1019 } else {
1020 SizeInChars = CGF.getTypeSize(
1021 SharedAddresses[N].first.getType().getNonReferenceType());
1022 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1023 }
1024 Sizes.emplace_back(SizeInChars, Size);
1025 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1026 CGF,
1027 cast<OpaqueValueExpr>(
1028 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1029 RValue::get(Size));
1030 CGF.EmitVariablyModifiedType(PrivateType);
1031}
1032
1033void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1034 llvm::Value *Size) {
1035 const auto *PrivateVD =
1036 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1037 QualType PrivateType = PrivateVD->getType();
1038 if (!PrivateType->isVariablyModifiedType()) {
1039 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1040 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1041 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
;
1042 return;
1043 }
1044 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1045 CGF,
1046 cast<OpaqueValueExpr>(
1047 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1048 RValue::get(Size));
1049 CGF.EmitVariablyModifiedType(PrivateType);
1050}
1051
1052void ReductionCodeGen::emitInitialization(
1053 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1054 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1055 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1055, __PRETTY_FUNCTION__))
;
1056 const auto *PrivateVD =
1057 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1058 const OMPDeclareReductionDecl *DRD =
1059 getReductionInit(ClausesData[N].ReductionOp);
1060 QualType PrivateType = PrivateVD->getType();
1061 PrivateAddr = CGF.Builder.CreateElementBitCast(
1062 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1063 QualType SharedType = SharedAddresses[N].first.getType();
1064 SharedLVal = CGF.MakeAddrLValue(
1065 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1066 CGF.ConvertTypeForMem(SharedType)),
1067 SharedType, SharedAddresses[N].first.getBaseInfo(),
1068 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1069 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1070 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1071 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1072 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1073 PrivateAddr, SharedLVal.getAddress(),
1074 SharedLVal.getType());
1075 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1076 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1077 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1078 PrivateVD->getType().getQualifiers(),
1079 /*IsInitializer=*/false);
1080 }
1081}
1082
1083bool ReductionCodeGen::needCleanups(unsigned N) {
1084 const auto *PrivateVD =
1085 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1086 QualType PrivateType = PrivateVD->getType();
1087 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1088 return DTorKind != QualType::DK_none;
1089}
1090
1091void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1092 Address PrivateAddr) {
1093 const auto *PrivateVD =
1094 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1095 QualType PrivateType = PrivateVD->getType();
1096 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1097 if (needCleanups(N)) {
1098 PrivateAddr = CGF.Builder.CreateElementBitCast(
1099 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1100 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1101 }
1102}
1103
1104static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1105 LValue BaseLV) {
1106 BaseTy = BaseTy.getNonReferenceType();
1107 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1108 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1109 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1110 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1111 } else {
1112 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1113 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1114 }
1115 BaseTy = BaseTy->getPointeeType();
1116 }
1117 return CGF.MakeAddrLValue(
1118 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1119 CGF.ConvertTypeForMem(ElTy)),
1120 BaseLV.getType(), BaseLV.getBaseInfo(),
1121 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1122}
1123
1124static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1125 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1126 llvm::Value *Addr) {
1127 Address Tmp = Address::invalid();
1128 Address TopTmp = Address::invalid();
1129 Address MostTopTmp = Address::invalid();
1130 BaseTy = BaseTy.getNonReferenceType();
1131 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1132 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1133 Tmp = CGF.CreateMemTemp(BaseTy);
1134 if (TopTmp.isValid())
1135 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1136 else
1137 MostTopTmp = Tmp;
1138 TopTmp = Tmp;
1139 BaseTy = BaseTy->getPointeeType();
1140 }
1141 llvm::Type *Ty = BaseLVType;
1142 if (Tmp.isValid())
1143 Ty = Tmp.getElementType();
1144 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1145 if (Tmp.isValid()) {
1146 CGF.Builder.CreateStore(Addr, Tmp);
1147 return MostTopTmp;
1148 }
1149 return Address(Addr, BaseLVAlignment);
1150}
1151
1152static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1153 const VarDecl *OrigVD = nullptr;
1154 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1155 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1156 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1157 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1158 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1159 Base = TempASE->getBase()->IgnoreParenImpCasts();
1160 DE = cast<DeclRefExpr>(Base);
1161 OrigVD = cast<VarDecl>(DE->getDecl());
1162 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1163 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1164 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1165 Base = TempASE->getBase()->IgnoreParenImpCasts();
1166 DE = cast<DeclRefExpr>(Base);
1167 OrigVD = cast<VarDecl>(DE->getDecl());
1168 }
1169 return OrigVD;
1170}
1171
1172Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1173 Address PrivateAddr) {
1174 const DeclRefExpr *DE;
1175 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1176 BaseDecls.emplace_back(OrigVD);
1177 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1178 LValue BaseLValue =
1179 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1180 OriginalBaseLValue);
1181 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1182 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1183 llvm::Value *PrivatePointer =
1184 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1185 PrivateAddr.getPointer(),
1186 SharedAddresses[N].first.getAddress().getType());
1187 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1188 return castToBase(CGF, OrigVD->getType(),
1189 SharedAddresses[N].first.getType(),
1190 OriginalBaseLValue.getAddress().getType(),
1191 OriginalBaseLValue.getAlignment(), Ptr);
1192 }
1193 BaseDecls.emplace_back(
1194 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1195 return PrivateAddr;
1196}
1197
1198bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1199 const OMPDeclareReductionDecl *DRD =
1200 getReductionInit(ClausesData[N].ReductionOp);
1201 return DRD && DRD->getInitializer();
1202}
1203
1204LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1205 return CGF.EmitLoadOfPointerLValue(
1206 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1207 getThreadIDVariable()->getType()->castAs<PointerType>());
1208}
1209
1210void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1211 if (!CGF.HaveInsertPoint())
1212 return;
1213 // 1.2.2 OpenMP Language Terminology
1214 // Structured block - An executable statement with a single entry at the
1215 // top and a single exit at the bottom.
1216 // The point of exit cannot be a branch out of the structured block.
1217 // longjmp() and throw() must not violate the entry/exit criteria.
1218 CGF.EHStack.pushTerminate();
1219 CodeGen(CGF);
1220 CGF.EHStack.popTerminate();
1221}
1222
1223LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1224 CodeGenFunction &CGF) {
1225 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1226 getThreadIDVariable()->getType(),
1227 AlignmentSource::Decl);
1228}
1229
1230static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1231 QualType FieldTy) {
1232 auto *Field = FieldDecl::Create(
1233 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1234 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1235 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1236 Field->setAccess(AS_public);
1237 DC->addDecl(Field);
1238 return Field;
1239}
1240
1241CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1242 StringRef Separator)
1243 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1244 OffloadEntriesInfoManager(CGM) {
1245 ASTContext &C = CGM.getContext();
1246 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1247 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1248 RD->startDefinition();
1249 // reserved_1
1250 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1251 // flags
1252 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1253 // reserved_2
1254 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1255 // reserved_3
1256 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1257 // psource
1258 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1259 RD->completeDefinition();
1260 IdentQTy = C.getRecordType(RD);
1261 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1262 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1263
1264 loadOffloadInfoMetadata();
1265}
1266
1267bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1268 const GlobalDecl &OldGD,
1269 llvm::GlobalValue *OrigAddr,
1270 bool IsForDefinition) {
1271 // Emit at least a definition for the aliasee if the the address of the
1272 // original function is requested.
1273 if (IsForDefinition || OrigAddr)
1274 (void)CGM.GetAddrOfGlobal(NewGD);
1275 StringRef NewMangledName = CGM.getMangledName(NewGD);
1276 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1277 if (Addr && !Addr->isDeclaration()) {
1278 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1279 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD);
1280 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1281
1282 // Create a reference to the named value. This ensures that it is emitted
1283 // if a deferred decl.
1284 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1285
1286 // Create the new alias itself, but don't set a name yet.
1287 auto *GA =
1288 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1289
1290 if (OrigAddr) {
1291 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1291, __PRETTY_FUNCTION__))
;
1292
1293 GA->takeName(OrigAddr);
1294 OrigAddr->replaceAllUsesWith(
1295 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1296 OrigAddr->eraseFromParent();
1297 } else {
1298 GA->setName(CGM.getMangledName(OldGD));
1299 }
1300
1301 // Set attributes which are particular to an alias; this is a
1302 // specialization of the attributes which may be set on a global function.
1303 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1304 D->isWeakImported())
1305 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1306
1307 CGM.SetCommonAttributes(OldGD, GA);
1308 return true;
1309 }
1310 return false;
1311}
1312
1313void CGOpenMPRuntime::clear() {
1314 InternalVars.clear();
1315 // Clean non-target variable declarations possibly used only in debug info.
1316 for (const auto &Data : EmittedNonTargetVariables) {
1317 if (!Data.getValue().pointsToAliveValue())
1318 continue;
1319 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1320 if (!GV)
1321 continue;
1322 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1323 continue;
1324 GV->eraseFromParent();
1325 }
1326 // Emit aliases for the deferred aliasees.
1327 for (const auto &Pair : DeferredVariantFunction) {
1328 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1329 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1330 // If not able to emit alias, just emit original declaration.
1331 (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1332 /*IsForDefinition=*/false);
1333 }
1334}
1335
1336std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1337 SmallString<128> Buffer;
1338 llvm::raw_svector_ostream OS(Buffer);
1339 StringRef Sep = FirstSeparator;
1340 for (StringRef Part : Parts) {
1341 OS << Sep << Part;
1342 Sep = Separator;
1343 }
1344 return OS.str();
1345}
1346
1347static llvm::Function *
1348emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1349 const Expr *CombinerInitializer, const VarDecl *In,
1350 const VarDecl *Out, bool IsCombiner) {
1351 // void .omp_combiner.(Ty *in, Ty *out);
1352 ASTContext &C = CGM.getContext();
1353 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1354 FunctionArgList Args;
1355 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1356 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1357 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1358 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1359 Args.push_back(&OmpOutParm);
1360 Args.push_back(&OmpInParm);
1361 const CGFunctionInfo &FnInfo =
1362 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1363 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1364 std::string Name = CGM.getOpenMPRuntime().getName(
1365 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1366 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1367 Name, &CGM.getModule());
1368 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1369 if (CGM.getLangOpts().Optimize) {
1370 Fn->removeFnAttr(llvm::Attribute::NoInline);
1371 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1372 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1373 }
1374 CodeGenFunction CGF(CGM);
1375 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1376 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1377 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1378 Out->getLocation());
1379 CodeGenFunction::OMPPrivateScope Scope(CGF);
1380 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1381 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1382 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1383 .getAddress();
1384 });
1385 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1386 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1387 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1388 .getAddress();
1389 });
1390 (void)Scope.Privatize();
1391 if (!IsCombiner && Out->hasInit() &&
1392 !CGF.isTrivialInitializer(Out->getInit())) {
1393 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1394 Out->getType().getQualifiers(),
1395 /*IsInitializer=*/true);
1396 }
1397 if (CombinerInitializer)
1398 CGF.EmitIgnoredExpr(CombinerInitializer);
1399 Scope.ForceCleanup();
1400 CGF.FinishFunction();
1401 return Fn;
1402}
1403
1404void CGOpenMPRuntime::emitUserDefinedReduction(
1405 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1406 if (UDRMap.count(D) > 0)
1407 return;
1408 llvm::Function *Combiner = emitCombinerOrInitializer(
1409 CGM, D->getType(), D->getCombiner(),
1410 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1411 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1412 /*IsCombiner=*/true);
1413 llvm::Function *Initializer = nullptr;
1414 if (const Expr *Init = D->getInitializer()) {
1415 Initializer = emitCombinerOrInitializer(
1416 CGM, D->getType(),
1417 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1418 : nullptr,
1419 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1420 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1421 /*IsCombiner=*/false);
1422 }
1423 UDRMap.try_emplace(D, Combiner, Initializer);
1424 if (CGF) {
1425 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1426 Decls.second.push_back(D);
1427 }
1428}
1429
1430std::pair<llvm::Function *, llvm::Function *>
1431CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1432 auto I = UDRMap.find(D);
1433 if (I != UDRMap.end())
1434 return I->second;
1435 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1436 return UDRMap.lookup(D);
1437}
1438
1439static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1440 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1441 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1442 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1443 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1444, __PRETTY_FUNCTION__))
1444 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1444, __PRETTY_FUNCTION__))
;
1445 CodeGenFunction CGF(CGM, true);
1446 bool HasCancel = false;
1447 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1448 HasCancel = OPD->hasCancel();
1449 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1450 HasCancel = OPSD->hasCancel();
1451 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1452 HasCancel = OPFD->hasCancel();
1453 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1454 HasCancel = OPFD->hasCancel();
1455 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1456 HasCancel = OPFD->hasCancel();
1457 else if (const auto *OPFD =
1458 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1459 HasCancel = OPFD->hasCancel();
1460 else if (const auto *OPFD =
1461 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1462 HasCancel = OPFD->hasCancel();
1463 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1464 HasCancel, OutlinedHelperName);
1465 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1466 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1467}
1468
1469llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1470 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1471 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1472 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1473 return emitParallelOrTeamsOutlinedFunction(
1474 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1475}
1476
1477llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1478 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1479 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1480 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1481 return emitParallelOrTeamsOutlinedFunction(
1482 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1483}
1484
1485llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1486 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1487 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1488 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1489 bool Tied, unsigned &NumberOfParts) {
1490 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1491 PrePostActionTy &) {
1492 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1493 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1494 llvm::Value *TaskArgs[] = {
1495 UpLoc, ThreadID,
1496 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1497 TaskTVar->getType()->castAs<PointerType>())
1498 .getPointer()};
1499 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1500 };
1501 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1502 UntiedCodeGen);
1503 CodeGen.setAction(Action);
1504 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1505, __PRETTY_FUNCTION__))
1505 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1505, __PRETTY_FUNCTION__))
;
1506 const OpenMPDirectiveKind Region =
1507 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1508 : OMPD_task;
1509 const CapturedStmt *CS = D.getCapturedStmt(Region);
1510 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1511 CodeGenFunction CGF(CGM, true);
1512 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1513 InnermostKind,
1514 TD ? TD->hasCancel() : false, Action);
1515 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1516 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1517 if (!Tied)
1518 NumberOfParts = Action.getNumberOfParts();
1519 return Res;
1520}
1521
1522static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1523 const RecordDecl *RD, const CGRecordLayout &RL,
1524 ArrayRef<llvm::Constant *> Data) {
1525 llvm::StructType *StructTy = RL.getLLVMType();
1526 unsigned PrevIdx = 0;
1527 ConstantInitBuilder CIBuilder(CGM);
1528 auto DI = Data.begin();
1529 for (const FieldDecl *FD : RD->fields()) {
1530 unsigned Idx = RL.getLLVMFieldNo(FD);
1531 // Fill the alignment.
1532 for (unsigned I = PrevIdx; I < Idx; ++I)
1533 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1534 PrevIdx = Idx + 1;
1535 Fields.add(*DI);
1536 ++DI;
1537 }
1538}
1539
1540template <class... As>
1541static llvm::GlobalVariable *
1542createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1543 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1544 As &&... Args) {
1545 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1546 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1547 ConstantInitBuilder CIBuilder(CGM);
1548 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1549 buildStructValue(Fields, CGM, RD, RL, Data);
1550 return Fields.finishAndCreateGlobal(
1551 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1552 std::forward<As>(Args)...);
1553}
1554
1555template <typename T>
1556static void
1557createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1558 ArrayRef<llvm::Constant *> Data,
1559 T &Parent) {
1560 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1561 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1562 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1563 buildStructValue(Fields, CGM, RD, RL, Data);
1564 Fields.finishAndAddTo(Parent);
1565}
1566
1567Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1568 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1569 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1570 FlagsTy FlagsKey(Flags, Reserved2Flags);
1571 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1572 if (!Entry) {
1573 if (!DefaultOpenMPPSource) {
1574 // Initialize default location for psource field of ident_t structure of
1575 // all ident_t objects. Format is ";file;function;line;column;;".
1576 // Taken from
1577 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1578 DefaultOpenMPPSource =
1579 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1580 DefaultOpenMPPSource =
1581 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1582 }
1583
1584 llvm::Constant *Data[] = {
1585 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1586 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1587 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1588 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1589 llvm::GlobalValue *DefaultOpenMPLocation =
1590 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1591 llvm::GlobalValue::PrivateLinkage);
1592 DefaultOpenMPLocation->setUnnamedAddr(
1593 llvm::GlobalValue::UnnamedAddr::Global);
1594
1595 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1596 }
1597 return Address(Entry, Align);
1598}
1599
1600void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1601 bool AtCurrentPoint) {
1602 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1603 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1603, __PRETTY_FUNCTION__))
;
1604
1605 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1606 if (AtCurrentPoint) {
1607 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1608 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1609 } else {
1610 Elem.second.ServiceInsertPt =
1611 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1612 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1613 }
1614}
1615
1616void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1617 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1618 if (Elem.second.ServiceInsertPt) {
1619 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1620 Elem.second.ServiceInsertPt = nullptr;
1621 Ptr->eraseFromParent();
1622 }
1623}
1624
1625llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1626 SourceLocation Loc,
1627 unsigned Flags) {
1628 Flags |= OMP_IDENT_KMPC;
1629 // If no debug info is generated - return global default location.
1630 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1631 Loc.isInvalid())
1632 return getOrCreateDefaultLocation(Flags).getPointer();
1633
1634 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1634, __PRETTY_FUNCTION__))
;
1635
1636 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1637 Address LocValue = Address::invalid();
1638 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1639 if (I != OpenMPLocThreadIDMap.end())
1640 LocValue = Address(I->second.DebugLoc, Align);
1641
1642 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1643 // GetOpenMPThreadID was called before this routine.
1644 if (!LocValue.isValid()) {
1645 // Generate "ident_t .kmpc_loc.addr;"
1646 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1647 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1648 Elem.second.DebugLoc = AI.getPointer();
1649 LocValue = AI;
1650
1651 if (!Elem.second.ServiceInsertPt)
1652 setLocThreadIdInsertPt(CGF);
1653 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1654 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1655 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1656 CGF.getTypeSize(IdentQTy));
1657 }
1658
1659 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1660 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1661 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1662 LValue PSource =
1663 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1664
1665 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1666 if (OMPDebugLoc == nullptr) {
1667 SmallString<128> Buffer2;
1668 llvm::raw_svector_ostream OS2(Buffer2);
1669 // Build debug location
1670 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1671 OS2 << ";" << PLoc.getFilename() << ";";
1672 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1673 OS2 << FD->getQualifiedNameAsString();
1674 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1675 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1676 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1677 }
1678 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1679 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1680
1681 // Our callers always pass this to a runtime function, so for
1682 // convenience, go ahead and return a naked pointer.
1683 return LocValue.getPointer();
1684}
1685
1686llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1687 SourceLocation Loc) {
1688 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1688, __PRETTY_FUNCTION__))
;
1689
1690 llvm::Value *ThreadID = nullptr;
1691 // Check whether we've already cached a load of the thread id in this
1692 // function.
1693 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1694 if (I != OpenMPLocThreadIDMap.end()) {
1695 ThreadID = I->second.ThreadID;
1696 if (ThreadID != nullptr)
1697 return ThreadID;
1698 }
1699 // If exceptions are enabled, do not use parameter to avoid possible crash.
1700 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1701 !CGF.getLangOpts().CXXExceptions ||
1702 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1703 if (auto *OMPRegionInfo =
1704 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1705 if (OMPRegionInfo->getThreadIDVariable()) {
1706 // Check if this an outlined function with thread id passed as argument.
1707 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1708 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1709 // If value loaded in entry block, cache it and use it everywhere in
1710 // function.
1711 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1712 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1713 Elem.second.ThreadID = ThreadID;
1714 }
1715 return ThreadID;
1716 }
1717 }
1718 }
1719
1720 // This is not an outlined function region - need to call __kmpc_int32
1721 // kmpc_global_thread_num(ident_t *loc).
1722 // Generate thread id value and cache this value for use across the
1723 // function.
1724 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1725 if (!Elem.second.ServiceInsertPt)
1726 setLocThreadIdInsertPt(CGF);
1727 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1728 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1729 llvm::CallInst *Call = CGF.Builder.CreateCall(
1730 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1731 emitUpdateLocation(CGF, Loc));
1732 Call->setCallingConv(CGF.getRuntimeCC());
1733 Elem.second.ThreadID = Call;
1734 return Call;
1735}
1736
1737void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1738 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1738, __PRETTY_FUNCTION__))
;
1739 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1740 clearLocThreadIdInsertPt(CGF);
1741 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1742 }
1743 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1744 for(auto *D : FunctionUDRMap[CGF.CurFn])
1745 UDRMap.erase(D);
1746 FunctionUDRMap.erase(CGF.CurFn);
1747 }
1748 auto I = FunctionUDMMap.find(CGF.CurFn);
1749 if (I != FunctionUDMMap.end()) {
1750 for(auto *D : I->second)
1751 UDMMap.erase(D);
1752 FunctionUDMMap.erase(I);
1753 }
1754}
1755
1756llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1757 return IdentTy->getPointerTo();
1758}
1759
1760llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1761 if (!Kmpc_MicroTy) {
1762 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1763 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1764 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1765 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1766 }
1767 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1768}
1769
1770llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1771 llvm::FunctionCallee RTLFn = nullptr;
1772 switch (static_cast<OpenMPRTLFunction>(Function)) {
1773 case OMPRTL__kmpc_fork_call: {
1774 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1775 // microtask, ...);
1776 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1777 getKmpc_MicroPointerTy()};
1778 auto *FnTy =
1779 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1780 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1781 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1782 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1783 llvm::LLVMContext &Ctx = F->getContext();
1784 llvm::MDBuilder MDB(Ctx);
1785 // Annotate the callback behavior of the __kmpc_fork_call:
1786 // - The callback callee is argument number 2 (microtask).
1787 // - The first two arguments of the callback callee are unknown (-1).
1788 // - All variadic arguments to the __kmpc_fork_call are passed to the
1789 // callback callee.
1790 F->addMetadata(
1791 llvm::LLVMContext::MD_callback,
1792 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1793 2, {-1, -1},
1794 /* VarArgsArePassed */ true)}));
1795 }
1796 }
1797 break;
1798 }
1799 case OMPRTL__kmpc_global_thread_num: {
1800 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1801 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1802 auto *FnTy =
1803 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1804 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1805 break;
1806 }
1807 case OMPRTL__kmpc_threadprivate_cached: {
1808 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1809 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1810 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1811 CGM.VoidPtrTy, CGM.SizeTy,
1812 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1813 auto *FnTy =
1814 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1815 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1816 break;
1817 }
1818 case OMPRTL__kmpc_critical: {
1819 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1820 // kmp_critical_name *crit);
1821 llvm::Type *TypeParams[] = {
1822 getIdentTyPointerTy(), CGM.Int32Ty,
1823 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1824 auto *FnTy =
1825 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1826 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1827 break;
1828 }
1829 case OMPRTL__kmpc_critical_with_hint: {
1830 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1831 // kmp_critical_name *crit, uintptr_t hint);
1832 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1833 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1834 CGM.IntPtrTy};
1835 auto *FnTy =
1836 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1837 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1838 break;
1839 }
1840 case OMPRTL__kmpc_threadprivate_register: {
1841 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1842 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1843 // typedef void *(*kmpc_ctor)(void *);
1844 auto *KmpcCtorTy =
1845 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1846 /*isVarArg*/ false)->getPointerTo();
1847 // typedef void *(*kmpc_cctor)(void *, void *);
1848 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1849 auto *KmpcCopyCtorTy =
1850 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1851 /*isVarArg*/ false)
1852 ->getPointerTo();
1853 // typedef void (*kmpc_dtor)(void *);
1854 auto *KmpcDtorTy =
1855 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1856 ->getPointerTo();
1857 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1858 KmpcCopyCtorTy, KmpcDtorTy};
1859 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1860 /*isVarArg*/ false);
1861 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1862 break;
1863 }
1864 case OMPRTL__kmpc_end_critical: {
1865 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1866 // kmp_critical_name *crit);
1867 llvm::Type *TypeParams[] = {
1868 getIdentTyPointerTy(), CGM.Int32Ty,
1869 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1870 auto *FnTy =
1871 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1872 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1873 break;
1874 }
1875 case OMPRTL__kmpc_cancel_barrier: {
1876 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1877 // global_tid);
1878 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1879 auto *FnTy =
1880 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1881 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1882 break;
1883 }
1884 case OMPRTL__kmpc_barrier: {
1885 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1886 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1887 auto *FnTy =
1888 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1889 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1890 break;
1891 }
1892 case OMPRTL__kmpc_for_static_fini: {
1893 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1894 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1895 auto *FnTy =
1896 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1897 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1898 break;
1899 }
1900 case OMPRTL__kmpc_push_num_threads: {
1901 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1902 // kmp_int32 num_threads)
1903 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1904 CGM.Int32Ty};
1905 auto *FnTy =
1906 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1907 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1908 break;
1909 }
1910 case OMPRTL__kmpc_serialized_parallel: {
1911 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1912 // global_tid);
1913 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1914 auto *FnTy =
1915 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1916 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1917 break;
1918 }
1919 case OMPRTL__kmpc_end_serialized_parallel: {
1920 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1921 // global_tid);
1922 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1923 auto *FnTy =
1924 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1925 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1926 break;
1927 }
1928 case OMPRTL__kmpc_flush: {
1929 // Build void __kmpc_flush(ident_t *loc);
1930 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1931 auto *FnTy =
1932 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1933 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1934 break;
1935 }
1936 case OMPRTL__kmpc_master: {
1937 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1938 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1939 auto *FnTy =
1940 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1941 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1942 break;
1943 }
1944 case OMPRTL__kmpc_end_master: {
1945 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1946 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1947 auto *FnTy =
1948 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1949 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1950 break;
1951 }
1952 case OMPRTL__kmpc_omp_taskyield: {
1953 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1954 // int end_part);
1955 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1956 auto *FnTy =
1957 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1958 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1959 break;
1960 }
1961 case OMPRTL__kmpc_single: {
1962 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1963 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1964 auto *FnTy =
1965 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1966 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1967 break;
1968 }
1969 case OMPRTL__kmpc_end_single: {
1970 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1971 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1972 auto *FnTy =
1973 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1974 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1975 break;
1976 }
1977 case OMPRTL__kmpc_omp_task_alloc: {
1978 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1979 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1980 // kmp_routine_entry_t *task_entry);
1981 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1982, __PRETTY_FUNCTION__))
1982 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1982, __PRETTY_FUNCTION__))
;
1983 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1984 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1985 // Return void * and then cast to particular kmp_task_t type.
1986 auto *FnTy =
1987 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1988 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1989 break;
1990 }
1991 case OMPRTL__kmpc_omp_target_task_alloc: {
1992 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
1993 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1994 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
1995 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1996, __PRETTY_FUNCTION__))
1996 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1996, __PRETTY_FUNCTION__))
;
1997 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1998 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
1999 CGM.Int64Ty};
2000 // Return void * and then cast to particular kmp_task_t type.
2001 auto *FnTy =
2002 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2003 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2004 break;
2005 }
2006 case OMPRTL__kmpc_omp_task: {
2007 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2008 // *new_task);
2009 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2010 CGM.VoidPtrTy};
2011 auto *FnTy =
2012 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2013 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2014 break;
2015 }
2016 case OMPRTL__kmpc_copyprivate: {
2017 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2018 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2019 // kmp_int32 didit);
2020 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2021 auto *CpyFnTy =
2022 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2023 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2024 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2025 CGM.Int32Ty};
2026 auto *FnTy =
2027 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2028 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2029 break;
2030 }
2031 case OMPRTL__kmpc_reduce: {
2032 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2033 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2034 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2035 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2036 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2037 /*isVarArg=*/false);
2038 llvm::Type *TypeParams[] = {
2039 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2040 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2041 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2042 auto *FnTy =
2043 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2044 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2045 break;
2046 }
2047 case OMPRTL__kmpc_reduce_nowait: {
2048 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2049 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2050 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2051 // *lck);
2052 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2053 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2054 /*isVarArg=*/false);
2055 llvm::Type *TypeParams[] = {
2056 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2057 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2058 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2059 auto *FnTy =
2060 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2061 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2062 break;
2063 }
2064 case OMPRTL__kmpc_end_reduce: {
2065 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2066 // kmp_critical_name *lck);
2067 llvm::Type *TypeParams[] = {
2068 getIdentTyPointerTy(), CGM.Int32Ty,
2069 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2070 auto *FnTy =
2071 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2072 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2073 break;
2074 }
2075 case OMPRTL__kmpc_end_reduce_nowait: {
2076 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2077 // kmp_critical_name *lck);
2078 llvm::Type *TypeParams[] = {
2079 getIdentTyPointerTy(), CGM.Int32Ty,
2080 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2081 auto *FnTy =
2082 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2083 RTLFn =
2084 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2085 break;
2086 }
2087 case OMPRTL__kmpc_omp_task_begin_if0: {
2088 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2089 // *new_task);
2090 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2091 CGM.VoidPtrTy};
2092 auto *FnTy =
2093 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2094 RTLFn =
2095 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2096 break;
2097 }
2098 case OMPRTL__kmpc_omp_task_complete_if0: {
2099 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2100 // *new_task);
2101 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2102 CGM.VoidPtrTy};
2103 auto *FnTy =
2104 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2105 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2106 /*Name=*/"__kmpc_omp_task_complete_if0");
2107 break;
2108 }
2109 case OMPRTL__kmpc_ordered: {
2110 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2111 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2112 auto *FnTy =
2113 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2114 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2115 break;
2116 }
2117 case OMPRTL__kmpc_end_ordered: {
2118 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2119 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2120 auto *FnTy =
2121 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2122 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2123 break;
2124 }
2125 case OMPRTL__kmpc_omp_taskwait: {
2126 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2127 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2128 auto *FnTy =
2129 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2130 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2131 break;
2132 }
2133 case OMPRTL__kmpc_taskgroup: {
2134 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2135 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2136 auto *FnTy =
2137 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2138 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2139 break;
2140 }
2141 case OMPRTL__kmpc_end_taskgroup: {
2142 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2143 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2144 auto *FnTy =
2145 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2146 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2147 break;
2148 }
2149 case OMPRTL__kmpc_push_proc_bind: {
2150 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2151 // int proc_bind)
2152 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2153 auto *FnTy =
2154 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2155 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2156 break;
2157 }
2158 case OMPRTL__kmpc_omp_task_with_deps: {
2159 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2160 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2161 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2162 llvm::Type *TypeParams[] = {
2163 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2164 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2165 auto *FnTy =
2166 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2167 RTLFn =
2168 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2169 break;
2170 }
2171 case OMPRTL__kmpc_omp_wait_deps: {
2172 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2173 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2174 // kmp_depend_info_t *noalias_dep_list);
2175 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2176 CGM.Int32Ty, CGM.VoidPtrTy,
2177 CGM.Int32Ty, CGM.VoidPtrTy};
2178 auto *FnTy =
2179 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2180 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2181 break;
2182 }
2183 case OMPRTL__kmpc_cancellationpoint: {
2184 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2185 // global_tid, kmp_int32 cncl_kind)
2186 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2187 auto *FnTy =
2188 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2189 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2190 break;
2191 }
2192 case OMPRTL__kmpc_cancel: {
2193 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2194 // kmp_int32 cncl_kind)
2195 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2196 auto *FnTy =
2197 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2198 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2199 break;
2200 }
2201 case OMPRTL__kmpc_push_num_teams: {
2202 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2203 // kmp_int32 num_teams, kmp_int32 num_threads)
2204 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2205 CGM.Int32Ty};
2206 auto *FnTy =
2207 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2208 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2209 break;
2210 }
2211 case OMPRTL__kmpc_fork_teams: {
2212 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2213 // microtask, ...);
2214 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2215 getKmpc_MicroPointerTy()};
2216 auto *FnTy =
2217 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2218 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2219 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2220 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2221 llvm::LLVMContext &Ctx = F->getContext();
2222 llvm::MDBuilder MDB(Ctx);
2223 // Annotate the callback behavior of the __kmpc_fork_teams:
2224 // - The callback callee is argument number 2 (microtask).
2225 // - The first two arguments of the callback callee are unknown (-1).
2226 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2227 // callback callee.
2228 F->addMetadata(
2229 llvm::LLVMContext::MD_callback,
2230 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2231 2, {-1, -1},
2232 /* VarArgsArePassed */ true)}));
2233 }
2234 }
2235 break;
2236 }
2237 case OMPRTL__kmpc_taskloop: {
2238 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2239 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2240 // sched, kmp_uint64 grainsize, void *task_dup);
2241 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2242 CGM.IntTy,
2243 CGM.VoidPtrTy,
2244 CGM.IntTy,
2245 CGM.Int64Ty->getPointerTo(),
2246 CGM.Int64Ty->getPointerTo(),
2247 CGM.Int64Ty,
2248 CGM.IntTy,
2249 CGM.IntTy,
2250 CGM.Int64Ty,
2251 CGM.VoidPtrTy};
2252 auto *FnTy =
2253 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2254 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2255 break;
2256 }
2257 case OMPRTL__kmpc_doacross_init: {
2258 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2259 // num_dims, struct kmp_dim *dims);
2260 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2261 CGM.Int32Ty,
2262 CGM.Int32Ty,
2263 CGM.VoidPtrTy};
2264 auto *FnTy =
2265 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2266 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2267 break;
2268 }
2269 case OMPRTL__kmpc_doacross_fini: {
2270 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2271 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2272 auto *FnTy =
2273 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2274 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2275 break;
2276 }
2277 case OMPRTL__kmpc_doacross_post: {
2278 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2279 // *vec);
2280 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2281 CGM.Int64Ty->getPointerTo()};
2282 auto *FnTy =
2283 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2284 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2285 break;
2286 }
2287 case OMPRTL__kmpc_doacross_wait: {
2288 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2289 // *vec);
2290 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2291 CGM.Int64Ty->getPointerTo()};
2292 auto *FnTy =
2293 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2294 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2295 break;
2296 }
2297 case OMPRTL__kmpc_task_reduction_init: {
2298 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2299 // *data);
2300 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2301 auto *FnTy =
2302 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2303 RTLFn =
2304 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2305 break;
2306 }
2307 case OMPRTL__kmpc_task_reduction_get_th_data: {
2308 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2309 // *d);
2310 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2311 auto *FnTy =
2312 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2313 RTLFn = CGM.CreateRuntimeFunction(
2314 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2315 break;
2316 }
2317 case OMPRTL__kmpc_alloc: {
2318 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2319 // al); omp_allocator_handle_t type is void *.
2320 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2321 auto *FnTy =
2322 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2323 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2324 break;
2325 }
2326 case OMPRTL__kmpc_free: {
2327 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2328 // al); omp_allocator_handle_t type is void *.
2329 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2330 auto *FnTy =
2331 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2332 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2333 break;
2334 }
2335 case OMPRTL__kmpc_push_target_tripcount: {
2336 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2337 // size);
2338 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2339 llvm::FunctionType *FnTy =
2340 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2341 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2342 break;
2343 }
2344 case OMPRTL__tgt_target: {
2345 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2346 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2347 // *arg_types);
2348 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2349 CGM.VoidPtrTy,
2350 CGM.Int32Ty,
2351 CGM.VoidPtrPtrTy,
2352 CGM.VoidPtrPtrTy,
2353 CGM.Int64Ty->getPointerTo(),
2354 CGM.Int64Ty->getPointerTo()};
2355 auto *FnTy =
2356 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2357 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2358 break;
2359 }
2360 case OMPRTL__tgt_target_nowait: {
2361 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2362 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2363 // int64_t *arg_types);
2364 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2365 CGM.VoidPtrTy,
2366 CGM.Int32Ty,
2367 CGM.VoidPtrPtrTy,
2368 CGM.VoidPtrPtrTy,
2369 CGM.Int64Ty->getPointerTo(),
2370 CGM.Int64Ty->getPointerTo()};
2371 auto *FnTy =
2372 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2373 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2374 break;
2375 }
2376 case OMPRTL__tgt_target_teams: {
2377 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2378 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2379 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2380 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2381 CGM.VoidPtrTy,
2382 CGM.Int32Ty,
2383 CGM.VoidPtrPtrTy,
2384 CGM.VoidPtrPtrTy,
2385 CGM.Int64Ty->getPointerTo(),
2386 CGM.Int64Ty->getPointerTo(),
2387 CGM.Int32Ty,
2388 CGM.Int32Ty};
2389 auto *FnTy =
2390 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2391 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2392 break;
2393 }
2394 case OMPRTL__tgt_target_teams_nowait: {
2395 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2396 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2397 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2398 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2399 CGM.VoidPtrTy,
2400 CGM.Int32Ty,
2401 CGM.VoidPtrPtrTy,
2402 CGM.VoidPtrPtrTy,
2403 CGM.Int64Ty->getPointerTo(),
2404 CGM.Int64Ty->getPointerTo(),
2405 CGM.Int32Ty,
2406 CGM.Int32Ty};
2407 auto *FnTy =
2408 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2409 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2410 break;
2411 }
2412 case OMPRTL__tgt_register_requires: {
2413 // Build void __tgt_register_requires(int64_t flags);
2414 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2415 auto *FnTy =
2416 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2417 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2418 break;
2419 }
2420 case OMPRTL__tgt_register_lib: {
2421 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2422 QualType ParamTy =
2423 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2424 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2425 auto *FnTy =
2426 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2427 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2428 break;
2429 }
2430 case OMPRTL__tgt_unregister_lib: {
2431 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2432 QualType ParamTy =
2433 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2434 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2435 auto *FnTy =
2436 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2437 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2438 break;
2439 }
2440 case OMPRTL__tgt_target_data_begin: {
2441 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2442 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2443 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2444 CGM.Int32Ty,
2445 CGM.VoidPtrPtrTy,
2446 CGM.VoidPtrPtrTy,
2447 CGM.Int64Ty->getPointerTo(),
2448 CGM.Int64Ty->getPointerTo()};
2449 auto *FnTy =
2450 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2451 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2452 break;
2453 }
2454 case OMPRTL__tgt_target_data_begin_nowait: {
2455 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2456 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2457 // *arg_types);
2458 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2459 CGM.Int32Ty,
2460 CGM.VoidPtrPtrTy,
2461 CGM.VoidPtrPtrTy,
2462 CGM.Int64Ty->getPointerTo(),
2463 CGM.Int64Ty->getPointerTo()};
2464 auto *FnTy =
2465 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2466 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2467 break;
2468 }
2469 case OMPRTL__tgt_target_data_end: {
2470 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2471 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2472 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2473 CGM.Int32Ty,
2474 CGM.VoidPtrPtrTy,
2475 CGM.VoidPtrPtrTy,
2476 CGM.Int64Ty->getPointerTo(),
2477 CGM.Int64Ty->getPointerTo()};
2478 auto *FnTy =
2479 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2480 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2481 break;
2482 }
2483 case OMPRTL__tgt_target_data_end_nowait: {
2484 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2485 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2486 // *arg_types);
2487 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2488 CGM.Int32Ty,
2489 CGM.VoidPtrPtrTy,
2490 CGM.VoidPtrPtrTy,
2491 CGM.Int64Ty->getPointerTo(),
2492 CGM.Int64Ty->getPointerTo()};
2493 auto *FnTy =
2494 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2495 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2496 break;
2497 }
2498 case OMPRTL__tgt_target_data_update: {
2499 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2500 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2501 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2502 CGM.Int32Ty,
2503 CGM.VoidPtrPtrTy,
2504 CGM.VoidPtrPtrTy,
2505 CGM.Int64Ty->getPointerTo(),
2506 CGM.Int64Ty->getPointerTo()};
2507 auto *FnTy =
2508 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2509 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2510 break;
2511 }
2512 case OMPRTL__tgt_target_data_update_nowait: {
2513 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2514 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2515 // *arg_types);
2516 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2517 CGM.Int32Ty,
2518 CGM.VoidPtrPtrTy,
2519 CGM.VoidPtrPtrTy,
2520 CGM.Int64Ty->getPointerTo(),
2521 CGM.Int64Ty->getPointerTo()};
2522 auto *FnTy =
2523 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2524 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2525 break;
2526 }
2527 case OMPRTL__tgt_mapper_num_components: {
2528 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2529 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2530 auto *FnTy =
2531 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2532 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2533 break;
2534 }
2535 case OMPRTL__tgt_push_mapper_component: {
2536 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2537 // *base, void *begin, int64_t size, int64_t type);
2538 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2539 CGM.Int64Ty, CGM.Int64Ty};
2540 auto *FnTy =
2541 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2542 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2543 break;
2544 }
2545 }
2546 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2546, __PRETTY_FUNCTION__))
;
2547 return RTLFn;
2548}
2549
2550llvm::FunctionCallee
2551CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2552 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2553, __PRETTY_FUNCTION__))
2553 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2553, __PRETTY_FUNCTION__))
;
2554 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2555 : "__kmpc_for_static_init_4u")
2556 : (IVSigned ? "__kmpc_for_static_init_8"
2557 : "__kmpc_for_static_init_8u");
2558 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2559 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2560 llvm::Type *TypeParams[] = {
2561 getIdentTyPointerTy(), // loc
2562 CGM.Int32Ty, // tid
2563 CGM.Int32Ty, // schedtype
2564 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2565 PtrTy, // p_lower
2566 PtrTy, // p_upper
2567 PtrTy, // p_stride
2568 ITy, // incr
2569 ITy // chunk
2570 };
2571 auto *FnTy =
2572 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2573 return CGM.CreateRuntimeFunction(FnTy, Name);
2574}
2575
2576llvm::FunctionCallee
2577CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2578 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2579, __PRETTY_FUNCTION__))
2579 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2579, __PRETTY_FUNCTION__))
;
2580 StringRef Name =
2581 IVSize == 32
2582 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2583 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2584 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2585 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2586 CGM.Int32Ty, // tid
2587 CGM.Int32Ty, // schedtype
2588 ITy, // lower
2589 ITy, // upper
2590 ITy, // stride
2591 ITy // chunk
2592 };
2593 auto *FnTy =
2594 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2595 return CGM.CreateRuntimeFunction(FnTy, Name);
2596}
2597
2598llvm::FunctionCallee
2599CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2600 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2601, __PRETTY_FUNCTION__))
2601 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2601, __PRETTY_FUNCTION__))
;
2602 StringRef Name =
2603 IVSize == 32
2604 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2605 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2606 llvm::Type *TypeParams[] = {
2607 getIdentTyPointerTy(), // loc
2608 CGM.Int32Ty, // tid
2609 };
2610 auto *FnTy =
2611 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2612 return CGM.CreateRuntimeFunction(FnTy, Name);
2613}
2614
2615llvm::FunctionCallee
2616CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2617 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2618, __PRETTY_FUNCTION__))
2618 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2618, __PRETTY_FUNCTION__))
;
2619 StringRef Name =
2620 IVSize == 32
2621 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2622 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2623 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2624 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2625 llvm::Type *TypeParams[] = {
2626 getIdentTyPointerTy(), // loc
2627 CGM.Int32Ty, // tid
2628 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2629 PtrTy, // p_lower
2630 PtrTy, // p_upper
2631 PtrTy // p_stride
2632 };
2633 auto *FnTy =
2634 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2635 return CGM.CreateRuntimeFunction(FnTy, Name);
2636}
2637
2638/// Obtain information that uniquely identifies a target entry. This
2639/// consists of the file and device IDs as well as line number associated with
2640/// the relevant entry source location.
2641static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2642 unsigned &DeviceID, unsigned &FileID,
2643 unsigned &LineNum) {
2644 SourceManager &SM = C.getSourceManager();
2645
2646 // The loc should be always valid and have a file ID (the user cannot use
2647 // #pragma directives in macros)
2648
2649 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2649, __PRETTY_FUNCTION__))
;
2650
2651 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2652 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2652, __PRETTY_FUNCTION__))
;
2653
2654 llvm::sys::fs::UniqueID ID;
2655 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2656 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2657 << PLoc.getFilename() << EC.message();
2658
2659 DeviceID = ID.getDevice();
2660 FileID = ID.getFile();
2661 LineNum = PLoc.getLine();
2662}
2663
2664Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2665 if (CGM.getLangOpts().OpenMPSimd)
2666 return Address::invalid();
2667 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2668 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2669 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2670 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2671 HasRequiresUnifiedSharedMemory))) {
2672 SmallString<64> PtrName;
2673 {
2674 llvm::raw_svector_ostream OS(PtrName);
2675 OS << CGM.getMangledName(GlobalDecl(VD));
2676 if (!VD->isExternallyVisible()) {
2677 unsigned DeviceID, FileID, Line;
2678 getTargetEntryUniqueInfo(CGM.getContext(),
2679 VD->getCanonicalDecl()->getBeginLoc(),
2680 DeviceID, FileID, Line);
2681 OS << llvm::format("_%x", FileID);
2682 }
2683 OS << "_decl_tgt_ref_ptr";
2684 }
2685 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2686 if (!Ptr) {
2687 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2688 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2689 PtrName);
2690
2691 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2692 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2693
2694 if (!CGM.getLangOpts().OpenMPIsDevice)
2695 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2696 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2697 }
2698 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2699 }
2700 return Address::invalid();
2701}
2702
2703llvm::Constant *
2704CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2705 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2706, __PRETTY_FUNCTION__))
2706 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2706, __PRETTY_FUNCTION__))
;
2707 // Lookup the entry, lazily creating it if necessary.
2708 std::string Suffix = getName({"cache", ""});
2709 return getOrCreateInternalVariable(
2710 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2711}
2712
2713Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2714 const VarDecl *VD,
2715 Address VDAddr,
2716 SourceLocation Loc) {
2717 if (CGM.getLangOpts().OpenMPUseTLS &&
2718 CGM.getContext().getTargetInfo().isTLSSupported())
2719 return VDAddr;
2720
2721 llvm::Type *VarTy = VDAddr.getElementType();
2722 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2723 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2724 CGM.Int8PtrTy),
2725 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2726 getOrCreateThreadPrivateCache(VD)};
2727 return Address(CGF.EmitRuntimeCall(
2728 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2729 VDAddr.getAlignment());
2730}
2731
2732void CGOpenMPRuntime::emitThreadPrivateVarInit(
2733 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2734 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2735 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2736 // library.
2737 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2738 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2739 OMPLoc);
2740 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2741 // to register constructor/destructor for variable.
2742 llvm::Value *Args[] = {
2743 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2744 Ctor, CopyCtor, Dtor};
2745 CGF.EmitRuntimeCall(
2746 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2747}
2748
2749llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2750 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2751 bool PerformInit, CodeGenFunction *CGF) {
2752 if (CGM.getLangOpts().OpenMPUseTLS &&
2753 CGM.getContext().getTargetInfo().isTLSSupported())
2754 return nullptr;
2755
2756 VD = VD->getDefinition(CGM.getContext());
2757 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2758 QualType ASTTy = VD->getType();
2759
2760 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2761 const Expr *Init = VD->getAnyInitializer();
2762 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2763 // Generate function that re-emits the declaration's initializer into the
2764 // threadprivate copy of the variable VD
2765 CodeGenFunction CtorCGF(CGM);
2766 FunctionArgList Args;
2767 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2768 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2769 ImplicitParamDecl::Other);
2770 Args.push_back(&Dst);
2771
2772 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2773 CGM.getContext().VoidPtrTy, Args);
2774 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2775 std::string Name = getName({"__kmpc_global_ctor_", ""});
2776 llvm::Function *Fn =
2777 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2778 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2779 Args, Loc, Loc);
2780 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2781 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2782 CGM.getContext().VoidPtrTy, Dst.getLocation());
2783 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2784 Arg = CtorCGF.Builder.CreateElementBitCast(
2785 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2786 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2787 /*IsInitializer=*/true);
2788 ArgVal = CtorCGF.EmitLoadOfScalar(
2789 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2790 CGM.getContext().VoidPtrTy, Dst.getLocation());
2791 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2792 CtorCGF.FinishFunction();
2793 Ctor = Fn;
2794 }
2795 if (VD->getType().isDestructedType() != QualType::DK_none) {
2796 // Generate function that emits destructor call for the threadprivate copy
2797 // of the variable VD
2798 CodeGenFunction DtorCGF(CGM);
2799 FunctionArgList Args;
2800 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2801 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2802 ImplicitParamDecl::Other);
2803 Args.push_back(&Dst);
2804
2805 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2806 CGM.getContext().VoidTy, Args);
2807 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2808 std::string Name = getName({"__kmpc_global_dtor_", ""});
2809 llvm::Function *Fn =
2810 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2811 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2812 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2813 Loc, Loc);
2814 // Create a scope with an artificial location for the body of this function.
2815 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2816 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2817 DtorCGF.GetAddrOfLocalVar(&Dst),
2818 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2819 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2820 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2821 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2822 DtorCGF.FinishFunction();
2823 Dtor = Fn;
2824 }
2825 // Do not emit init function if it is not required.
2826 if (!Ctor && !Dtor)
2827 return nullptr;
2828
2829 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2830 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2831 /*isVarArg=*/false)
2832 ->getPointerTo();
2833 // Copying constructor for the threadprivate variable.
2834 // Must be NULL - reserved by runtime, but currently it requires that this
2835 // parameter is always NULL. Otherwise it fires assertion.
2836 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2837 if (Ctor == nullptr) {
2838 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2839 /*isVarArg=*/false)
2840 ->getPointerTo();
2841 Ctor = llvm::Constant::getNullValue(CtorTy);
2842 }
2843 if (Dtor == nullptr) {
2844 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2845 /*isVarArg=*/false)
2846 ->getPointerTo();
2847 Dtor = llvm::Constant::getNullValue(DtorTy);
2848 }
2849 if (!CGF) {
2850 auto *InitFunctionTy =
2851 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2852 std::string Name = getName({"__omp_threadprivate_init_", ""});
2853 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2854 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2855 CodeGenFunction InitCGF(CGM);
2856 FunctionArgList ArgList;
2857 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2858 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2859 Loc, Loc);
2860 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2861 InitCGF.FinishFunction();
2862 return InitFunction;
2863 }
2864 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2865 }
2866 return nullptr;
2867}
2868
2869bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2870 llvm::GlobalVariable *Addr,
2871 bool PerformInit) {
2872 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2873 !CGM.getLangOpts().OpenMPIsDevice)
2874 return false;
2875 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2876 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2877 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2878 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2879 HasRequiresUnifiedSharedMemory))
2880 return CGM.getLangOpts().OpenMPIsDevice;
2881 VD = VD->getDefinition(CGM.getContext());
2882 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2883 return CGM.getLangOpts().OpenMPIsDevice;
2884
2885 QualType ASTTy = VD->getType();
2886
2887 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2888 // Produce the unique prefix to identify the new target regions. We use
2889 // the source location of the variable declaration which we know to not
2890 // conflict with any target region.
2891 unsigned DeviceID;
2892 unsigned FileID;
2893 unsigned Line;
2894 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2895 SmallString<128> Buffer, Out;
2896 {
2897 llvm::raw_svector_ostream OS(Buffer);
2898 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2899 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2900 }
2901
2902 const Expr *Init = VD->getAnyInitializer();
2903 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2904 llvm::Constant *Ctor;
2905 llvm::Constant *ID;
2906 if (CGM.getLangOpts().OpenMPIsDevice) {
2907 // Generate function that re-emits the declaration's initializer into
2908 // the threadprivate copy of the variable VD
2909 CodeGenFunction CtorCGF(CGM);
2910
2911 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2912 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2913 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2914 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2915 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2916 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2917 FunctionArgList(), Loc, Loc);
2918 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2919 CtorCGF.EmitAnyExprToMem(Init,
2920 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2921 Init->getType().getQualifiers(),
2922 /*IsInitializer=*/true);
2923 CtorCGF.FinishFunction();
2924 Ctor = Fn;
2925 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2926 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2927 } else {
2928 Ctor = new llvm::GlobalVariable(
2929 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2930 llvm::GlobalValue::PrivateLinkage,
2931 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2932 ID = Ctor;
2933 }
2934
2935 // Register the information for the entry associated with the constructor.
2936 Out.clear();
2937 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2938 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2939 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2940 }
2941 if (VD->getType().isDestructedType() != QualType::DK_none) {
2942 llvm::Constant *Dtor;
2943 llvm::Constant *ID;
2944 if (CGM.getLangOpts().OpenMPIsDevice) {
2945 // Generate function that emits destructor call for the threadprivate
2946 // copy of the variable VD
2947 CodeGenFunction DtorCGF(CGM);
2948
2949 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2950 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2951 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2952 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2953 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2954 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2955 FunctionArgList(), Loc, Loc);
2956 // Create a scope with an artificial location for the body of this
2957 // function.
2958 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2959 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2960 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2961 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2962 DtorCGF.FinishFunction();
2963 Dtor = Fn;
2964 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2965 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2966 } else {
2967 Dtor = new llvm::GlobalVariable(
2968 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2969 llvm::GlobalValue::PrivateLinkage,
2970 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2971 ID = Dtor;
2972 }
2973 // Register the information for the entry associated with the destructor.
2974 Out.clear();
2975 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2976 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2977 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2978 }
2979 return CGM.getLangOpts().OpenMPIsDevice;
2980}
2981
2982Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2983 QualType VarType,
2984 StringRef Name) {
2985 std::string Suffix = getName({"artificial", ""});
2986 std::string CacheSuffix = getName({"cache", ""});
2987 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2988 llvm::Value *GAddr =
2989 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2990 llvm::Value *Args[] = {
2991 emitUpdateLocation(CGF, SourceLocation()),
2992 getThreadID(CGF, SourceLocation()),
2993 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2994 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2995 /*isSigned=*/false),
2996 getOrCreateInternalVariable(
2997 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2998 return Address(
2999 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3000 CGF.EmitRuntimeCall(
3001 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3002 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3003 CGM.getPointerAlign());
3004}
3005
3006void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
3007 const RegionCodeGenTy &ThenGen,
3008 const RegionCodeGenTy &ElseGen) {
3009 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3010
3011 // If the condition constant folds and can be elided, try to avoid emitting
3012 // the condition and the dead arm of the if/else.
3013 bool CondConstant;
3014 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
9
Assuming the condition is false
10
Taking false branch
3015 if (CondConstant)
3016 ThenGen(CGF);
3017 else
3018 ElseGen(CGF);
3019 return;
3020 }
3021
3022 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3023 // emit the conditional branch.
3024 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3025 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3026 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3027 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3028
3029 // Emit the 'then' code.
3030 CGF.EmitBlock(ThenBlock);
3031 ThenGen(CGF);
11
Calling 'RegionCodeGenTy::operator()'
3032 CGF.EmitBranch(ContBlock);
3033 // Emit the 'else' code if present.
3034 // There is no need to emit line number for unconditional branch.
3035 (void)ApplyDebugLocation::CreateEmpty(CGF);
3036 CGF.EmitBlock(ElseBlock);
3037 ElseGen(CGF);
3038 // There is no need to emit line number for unconditional branch.
3039 (void)ApplyDebugLocation::CreateEmpty(CGF);
3040 CGF.EmitBranch(ContBlock);
3041 // Emit the continuation block for code after the if.
3042 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3043}
3044
3045void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3046 llvm::Function *OutlinedFn,
3047 ArrayRef<llvm::Value *> CapturedVars,
3048 const Expr *IfCond) {
3049 if (!CGF.HaveInsertPoint())
3050 return;
3051 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3052 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3053 PrePostActionTy &) {
3054 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3055 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3056 llvm::Value *Args[] = {
3057 RTLoc,
3058 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3059 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3060 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3061 RealArgs.append(std::begin(Args), std::end(Args));
3062 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3063
3064 llvm::FunctionCallee RTLFn =
3065 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3066 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3067 };
3068 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3069 PrePostActionTy &) {
3070 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3071 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3072 // Build calls:
3073 // __kmpc_serialized_parallel(&Loc, GTid);
3074 llvm::Value *Args[] = {RTLoc, ThreadID};
3075 CGF.EmitRuntimeCall(
3076 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3077
3078 // OutlinedFn(&GTid, &zero, CapturedStruct);
3079 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3080 /*Name*/ ".zero.addr");
3081 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
3082 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3083 // ThreadId for serialized parallels is 0.
3084 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3085 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3086 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3087 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3088
3089 // __kmpc_end_serialized_parallel(&Loc, GTid);
3090 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3091 CGF.EmitRuntimeCall(
3092 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3093 EndArgs);
3094 };
3095 if (IfCond) {
3096 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
3097 } else {
3098 RegionCodeGenTy ThenRCG(ThenGen);
3099 ThenRCG(CGF);
3100 }
3101}
3102
3103// If we're inside an (outlined) parallel region, use the region info's
3104// thread-ID variable (it is passed in a first argument of the outlined function
3105// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3106// regular serial code region, get thread ID by calling kmp_int32
3107// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3108// return the address of that temp.
3109Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3110 SourceLocation Loc) {
3111 if (auto *OMPRegionInfo =
3112 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3113 if (OMPRegionInfo->getThreadIDVariable())
3114 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
3115
3116 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3117 QualType Int32Ty =
3118 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3119 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3120 CGF.EmitStoreOfScalar(ThreadID,
3121 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3122
3123 return ThreadIDTemp;
3124}
3125
3126llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3127 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3128 SmallString<256> Buffer;
3129 llvm::raw_svector_ostream Out(Buffer);
3130 Out << Name;
3131 StringRef RuntimeName = Out.str();
3132 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3133 if (Elem.second) {
3134 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3135, __PRETTY_FUNCTION__))
3135 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3135, __PRETTY_FUNCTION__))
;
3136 return &*Elem.second;
3137 }
3138
3139 return Elem.second = new llvm::GlobalVariable(
3140 CGM.getModule(), Ty, /*IsConstant*/ false,
3141 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3142 Elem.first(), /*InsertBefore=*/nullptr,
3143 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3144}
3145
3146llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3147 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3148 std::string Name = getName({Prefix, "var"});
3149 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3150}
3151
3152namespace {
3153/// Common pre(post)-action for different OpenMP constructs.
3154class CommonActionTy final : public PrePostActionTy {
3155 llvm::FunctionCallee EnterCallee;
3156 ArrayRef<llvm::Value *> EnterArgs;
3157 llvm::FunctionCallee ExitCallee;
3158 ArrayRef<llvm::Value *> ExitArgs;
3159 bool Conditional;
3160 llvm::BasicBlock *ContBlock = nullptr;
3161
3162public:
3163 CommonActionTy(llvm::FunctionCallee EnterCallee,
3164 ArrayRef<llvm::Value *> EnterArgs,
3165 llvm::FunctionCallee ExitCallee,
3166 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3167 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3168 ExitArgs(ExitArgs), Conditional(Conditional) {}
3169 void Enter(CodeGenFunction &CGF) override {
3170 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3171 if (Conditional) {
3172 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3173 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3174 ContBlock = CGF.createBasicBlock("omp_if.end");
3175 // Generate the branch (If-stmt)
3176 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3177 CGF.EmitBlock(ThenBlock);
3178 }
3179 }
3180 void Done(CodeGenFunction &CGF) {
3181 // Emit the rest of blocks/branches
3182 CGF.EmitBranch(ContBlock);
3183 CGF.EmitBlock(ContBlock, true);
3184 }
3185 void Exit(CodeGenFunction &CGF) override {
3186 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3187 }
3188};
3189} // anonymous namespace
3190
3191void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3192 StringRef CriticalName,
3193 const RegionCodeGenTy &CriticalOpGen,
3194 SourceLocation Loc, const Expr *Hint) {
3195 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3196 // CriticalOpGen();
3197 // __kmpc_end_critical(ident_t *, gtid, Lock);
3198 // Prepare arguments and build a call to __kmpc_critical
3199 if (!CGF.HaveInsertPoint())
3200 return;
3201 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3202 getCriticalRegionLock(CriticalName)};
3203 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3204 std::end(Args));
3205 if (Hint) {
3206 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3207 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3208 }
3209 CommonActionTy Action(
3210 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3211 : OMPRTL__kmpc_critical),
3212 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3213 CriticalOpGen.setAction(Action);
3214 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3215}
3216
3217void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3218 const RegionCodeGenTy &MasterOpGen,
3219 SourceLocation Loc) {
3220 if (!CGF.HaveInsertPoint())
3221 return;
3222 // if(__kmpc_master(ident_t *, gtid)) {
3223 // MasterOpGen();
3224 // __kmpc_end_master(ident_t *, gtid);
3225 // }
3226 // Prepare arguments and build a call to __kmpc_master
3227 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3228 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3229 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3230 /*Conditional=*/true);
3231 MasterOpGen.setAction(Action);
3232 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3233 Action.Done(CGF);
3234}
3235
3236void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3237 SourceLocation Loc) {
3238 if (!CGF.HaveInsertPoint())
3239 return;
3240 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3241 llvm::Value *Args[] = {
3242 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3243 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3244 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3245 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3246 Region->emitUntiedSwitch(CGF);
3247}
3248
3249void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3250 const RegionCodeGenTy &TaskgroupOpGen,
3251 SourceLocation Loc) {
3252 if (!CGF.HaveInsertPoint())
3253 return;
3254 // __kmpc_taskgroup(ident_t *, gtid);
3255 // TaskgroupOpGen();
3256 // __kmpc_end_taskgroup(ident_t *, gtid);
3257 // Prepare arguments and build a call to __kmpc_taskgroup
3258 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3259 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3260 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3261 Args);
3262 TaskgroupOpGen.setAction(Action);
3263 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3264}
3265
3266/// Given an array of pointers to variables, project the address of a
3267/// given variable.
3268static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3269 unsigned Index, const VarDecl *Var) {
3270 // Pull out the pointer to the variable.
3271 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3272 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3273
3274 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3275 Addr = CGF.Builder.CreateElementBitCast(
3276 Addr, CGF.ConvertTypeForMem(Var->getType()));
3277 return Addr;
3278}
3279
3280static llvm::Value *emitCopyprivateCopyFunction(
3281 CodeGenModule &CGM, llvm::Type *ArgsType,
3282 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3283 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3284 SourceLocation Loc) {
3285 ASTContext &C = CGM.getContext();
3286 // void copy_func(void *LHSArg, void *RHSArg);
3287 FunctionArgList Args;
3288 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3289 ImplicitParamDecl::Other);
3290 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3291 ImplicitParamDecl::Other);
3292 Args.push_back(&LHSArg);
3293 Args.push_back(&RHSArg);
3294 const auto &CGFI =
3295 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3296 std::string Name =
3297 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3298 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3299 llvm::GlobalValue::InternalLinkage, Name,
3300 &CGM.getModule());
3301 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3302 Fn->setDoesNotRecurse();
3303 CodeGenFunction CGF(CGM);
3304 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3305 // Dest = (void*[n])(LHSArg);
3306 // Src = (void*[n])(RHSArg);
3307 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3308 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3309 ArgsType), CGF.getPointerAlign());
3310 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3311 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3312 ArgsType), CGF.getPointerAlign());
3313 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3314 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3315 // ...
3316 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3317 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3318 const auto *DestVar =
3319 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3320 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3321
3322 const auto *SrcVar =
3323 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3324 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3325
3326 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3327 QualType Type = VD->getType();
3328 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3329 }
3330 CGF.FinishFunction();
3331 return Fn;
3332}
3333
3334void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3335 const RegionCodeGenTy &SingleOpGen,
3336 SourceLocation Loc,
3337 ArrayRef<const Expr *> CopyprivateVars,
3338 ArrayRef<const Expr *> SrcExprs,
3339 ArrayRef<const Expr *> DstExprs,
3340 ArrayRef<const Expr *> AssignmentOps) {
3341 if (!CGF.HaveInsertPoint())
3342 return;
3343 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3345, __PRETTY_FUNCTION__))
3344 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3345, __PRETTY_FUNCTION__))
3345 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3345, __PRETTY_FUNCTION__))
;
3346 ASTContext &C = CGM.getContext();
3347 // int32 did_it = 0;
3348 // if(__kmpc_single(ident_t *, gtid)) {
3349 // SingleOpGen();
3350 // __kmpc_end_single(ident_t *, gtid);
3351 // did_it = 1;
3352 // }
3353 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3354 // <copy_func>, did_it);
3355
3356 Address DidIt = Address::invalid();
3357 if (!CopyprivateVars.empty()) {
3358 // int32 did_it = 0;
3359 QualType KmpInt32Ty =
3360 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3361 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3362 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3363 }
3364 // Prepare arguments and build a call to __kmpc_single
3365 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3366 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3367 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3368 /*Conditional=*/true);
3369 SingleOpGen.setAction(Action);
3370 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3371 if (DidIt.isValid()) {
3372 // did_it = 1;
3373 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3374 }
3375 Action.Done(CGF);
3376 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3377 // <copy_func>, did_it);
3378 if (DidIt.isValid()) {
3379 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3380 QualType CopyprivateArrayTy = C.getConstantArrayType(
3381 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3382 /*IndexTypeQuals=*/0);
3383 // Create a list of all private variables for copyprivate.
3384 Address CopyprivateList =
3385 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3386 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3387 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3388 CGF.Builder.CreateStore(
3389 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3390 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3391 Elem);
3392 }
3393 // Build function that copies private values from single region to all other
3394 // threads in the corresponding parallel region.
3395 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3396 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3397 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3398 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3399 Address CL =
3400 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3401 CGF.VoidPtrTy);
3402 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3403 llvm::Value *Args[] = {
3404 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3405 getThreadID(CGF, Loc), // i32 <gtid>
3406 BufSize, // size_t <buf_size>
3407 CL.getPointer(), // void *<copyprivate list>
3408 CpyFn, // void (*) (void *, void *) <copy_func>
3409 DidItVal // i32 did_it
3410 };
3411 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3412 }
3413}
3414
3415void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3416 const RegionCodeGenTy &OrderedOpGen,
3417 SourceLocation Loc, bool IsThreads) {
3418 if (!CGF.HaveInsertPoint())
3419 return;
3420 // __kmpc_ordered(ident_t *, gtid);
3421 // OrderedOpGen();
3422 // __kmpc_end_ordered(ident_t *, gtid);
3423 // Prepare arguments and build a call to __kmpc_ordered
3424 if (IsThreads) {
3425 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3426 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3427 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3428 Args);
3429 OrderedOpGen.setAction(Action);
3430 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3431 return;
3432 }
3433 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3434}
3435
3436unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3437 unsigned Flags;
3438 if (Kind == OMPD_for)
3439 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3440 else if (Kind == OMPD_sections)
3441 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3442 else if (Kind == OMPD_single)
3443 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3444 else if (Kind == OMPD_barrier)
3445 Flags = OMP_IDENT_BARRIER_EXPL;
3446 else
3447 Flags = OMP_IDENT_BARRIER_IMPL;
3448 return Flags;
3449}
3450
3451void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3452 CodeGenFunction &CGF, const OMPLoopDirective &S,
3453 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3454 // Check if the loop directive is actually a doacross loop directive. In this
3455 // case choose static, 1 schedule.
3456 if (llvm::any_of(
3457 S.getClausesOfKind<OMPOrderedClause>(),
3458 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3459 ScheduleKind = OMPC_SCHEDULE_static;
3460 // Chunk size is 1 in this case.
3461 llvm::APInt ChunkSize(32, 1);
3462 ChunkExpr = IntegerLiteral::Create(
3463 CGF.getContext(), ChunkSize,
3464 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3465 SourceLocation());
3466 }
3467}
3468
3469void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3470 OpenMPDirectiveKind Kind, bool EmitChecks,
3471 bool ForceSimpleCall) {
3472 if (!CGF.HaveInsertPoint())
3473 return;
3474 // Build call __kmpc_cancel_barrier(loc, thread_id);
3475 // Build call __kmpc_barrier(loc, thread_id);
3476 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3477 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3478 // thread_id);
3479 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3480 getThreadID(CGF, Loc)};
3481 if (auto *OMPRegionInfo =
3482 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3483 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3484 llvm::Value *Result = CGF.EmitRuntimeCall(
3485 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3486 if (EmitChecks) {
3487 // if (__kmpc_cancel_barrier()) {
3488 // exit from construct;
3489 // }
3490 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3491 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3492 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3493 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3494 CGF.EmitBlock(ExitBB);
3495 // exit from construct;
3496 CodeGenFunction::JumpDest CancelDestination =
3497 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3498 CGF.EmitBranchThroughCleanup(CancelDestination);
3499 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3500 }
3501 return;
3502 }
3503 }
3504 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3505}
3506
3507/// Map the OpenMP loop schedule to the runtime enumeration.
3508static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3509 bool Chunked, bool Ordered) {
3510 switch (ScheduleKind) {
3511 case OMPC_SCHEDULE_static:
3512 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3513 : (Ordered ? OMP_ord_static : OMP_sch_static);
3514 case OMPC_SCHEDULE_dynamic:
3515 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3516 case OMPC_SCHEDULE_guided:
3517 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3518 case OMPC_SCHEDULE_runtime:
3519 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3520 case OMPC_SCHEDULE_auto:
3521 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3522 case OMPC_SCHEDULE_unknown:
3523 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3523, __PRETTY_FUNCTION__))
;
3524 return Ordered ? OMP_ord_static : OMP_sch_static;
3525 }
3526 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3526)
;
3527}
3528
3529/// Map the OpenMP distribute schedule to the runtime enumeration.
3530static OpenMPSchedType
3531getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3532 // only static is allowed for dist_schedule
3533 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3534}
3535
3536bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3537 bool Chunked) const {
3538 OpenMPSchedType Schedule =
3539 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3540 return Schedule == OMP_sch_static;
3541}
3542
3543bool CGOpenMPRuntime::isStaticNonchunked(
3544 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3545 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3546 return Schedule == OMP_dist_sch_static;
3547}
3548
3549bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3550 bool Chunked) const {
3551 OpenMPSchedType Schedule =
3552 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3553 return Schedule == OMP_sch_static_chunked;
3554}
3555
3556bool CGOpenMPRuntime::isStaticChunked(
3557 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3558 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3559 return Schedule == OMP_dist_sch_static_chunked;
3560}
3561
3562bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3563 OpenMPSchedType Schedule =
3564 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3565 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3565, __PRETTY_FUNCTION__))
;
3566 return Schedule != OMP_sch_static;
3567}
3568
3569static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3570 OpenMPScheduleClauseModifier M1,
3571 OpenMPScheduleClauseModifier M2) {
3572 int Modifier = 0;
3573 switch (M1) {
3574 case OMPC_SCHEDULE_MODIFIER_monotonic:
3575 Modifier = OMP_sch_modifier_monotonic;
3576 break;
3577 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3578 Modifier = OMP_sch_modifier_nonmonotonic;
3579 break;
3580 case OMPC_SCHEDULE_MODIFIER_simd:
3581 if (Schedule == OMP_sch_static_chunked)
3582 Schedule = OMP_sch_static_balanced_chunked;
3583 break;
3584 case OMPC_SCHEDULE_MODIFIER_last:
3585 case OMPC_SCHEDULE_MODIFIER_unknown:
3586 break;
3587 }
3588 switch (M2) {
3589 case OMPC_SCHEDULE_MODIFIER_monotonic:
3590 Modifier = OMP_sch_modifier_monotonic;
3591 break;
3592 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3593 Modifier = OMP_sch_modifier_nonmonotonic;
3594 break;
3595 case OMPC_SCHEDULE_MODIFIER_simd:
3596 if (Schedule == OMP_sch_static_chunked)
3597 Schedule = OMP_sch_static_balanced_chunked;
3598 break;
3599 case OMPC_SCHEDULE_MODIFIER_last:
3600 case OMPC_SCHEDULE_MODIFIER_unknown:
3601 break;
3602 }
3603 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3604 // If the static schedule kind is specified or if the ordered clause is
3605 // specified, and if the nonmonotonic modifier is not specified, the effect is
3606 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3607 // modifier is specified, the effect is as if the nonmonotonic modifier is
3608 // specified.
3609 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3610 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3611 Schedule == OMP_sch_static_balanced_chunked ||
3612 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static))
3613 Modifier = OMP_sch_modifier_nonmonotonic;
3614 }
3615 return Schedule | Modifier;
3616}
3617
3618void CGOpenMPRuntime::emitForDispatchInit(
3619 CodeGenFunction &CGF, SourceLocation Loc,
3620 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3621 bool Ordered, const DispatchRTInput &DispatchValues) {
3622 if (!CGF.HaveInsertPoint())
3623 return;
3624 OpenMPSchedType Schedule = getRuntimeSchedule(
3625 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3626 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3629, __PRETTY_FUNCTION__))
3627 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3629, __PRETTY_FUNCTION__))
3628 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3629, __PRETTY_FUNCTION__))
3629 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3629, __PRETTY_FUNCTION__))
;
3630 // Call __kmpc_dispatch_init(
3631 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3632 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3633 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3634
3635 // If the Chunk was not specified in the clause - use default value 1.
3636 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3637 : CGF.Builder.getIntN(IVSize, 1);
3638 llvm::Value *Args[] = {
3639 emitUpdateLocation(CGF, Loc),
3640 getThreadID(CGF, Loc),
3641 CGF.Builder.getInt32(addMonoNonMonoModifier(
3642 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3643 DispatchValues.LB, // Lower
3644 DispatchValues.UB, // Upper
3645 CGF.Builder.getIntN(IVSize, 1), // Stride
3646 Chunk // Chunk
3647 };
3648 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3649}
3650
3651static void emitForStaticInitCall(
3652 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3653 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3654 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3655 const CGOpenMPRuntime::StaticRTInput &Values) {
3656 if (!CGF.HaveInsertPoint())
3657 return;
3658
3659 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3659, __PRETTY_FUNCTION__))
;
3660 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3664, __PRETTY_FUNCTION__))
3661 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3664, __PRETTY_FUNCTION__))
3662 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3664, __PRETTY_FUNCTION__))
3663 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3664, __PRETTY_FUNCTION__))
3664 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3664, __PRETTY_FUNCTION__))
;
3665
3666 // Call __kmpc_for_static_init(
3667 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3668 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3669 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3670 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3671 llvm::Value *Chunk = Values.Chunk;
3672 if (Chunk == nullptr) {
3673 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3675, __PRETTY_FUNCTION__))
3674 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3675, __PRETTY_FUNCTION__))
3675 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3675, __PRETTY_FUNCTION__))
;
3676 // If the Chunk was not specified in the clause - use default value 1.
3677 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3678 } else {
3679 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3683, __PRETTY_FUNCTION__))
3680 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3683, __PRETTY_FUNCTION__))
3681 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3683, __PRETTY_FUNCTION__))
3682 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3683, __PRETTY_FUNCTION__))
3683 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3683, __PRETTY_FUNCTION__))
;
3684 }
3685 llvm::Value *Args[] = {
3686 UpdateLocation,
3687 ThreadId,
3688 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3689 M2)), // Schedule type
3690 Values.IL.getPointer(), // &isLastIter
3691 Values.LB.getPointer(), // &LB
3692 Values.UB.getPointer(), // &UB
3693 Values.ST.getPointer(), // &Stride
3694 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3695 Chunk // Chunk
3696 };
3697 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3698}
3699
3700void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3701 SourceLocation Loc,
3702 OpenMPDirectiveKind DKind,
3703 const OpenMPScheduleTy &ScheduleKind,
3704 const StaticRTInput &Values) {
3705 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3706 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3707 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
3708 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
;
3709 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3710 isOpenMPLoopDirective(DKind)
3711 ? OMP_IDENT_WORK_LOOP
3712 : OMP_IDENT_WORK_SECTIONS);
3713 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3714 llvm::FunctionCallee StaticInitFunction =
3715 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3716 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3717 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3718}
3719
3720void CGOpenMPRuntime::emitDistributeStaticInit(
3721 CodeGenFunction &CGF, SourceLocation Loc,
3722 OpenMPDistScheduleClauseKind SchedKind,
3723 const CGOpenMPRuntime::StaticRTInput &Values) {
3724 OpenMPSchedType ScheduleNum =
3725 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3726 llvm::Value *UpdatedLocation =
3727 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3728 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3729 llvm::FunctionCallee StaticInitFunction =
3730 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3731 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3732 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3733 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3734}
3735
3736void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3737 SourceLocation Loc,
3738 OpenMPDirectiveKind DKind) {
3739 if (!CGF.HaveInsertPoint())
3740 return;
3741 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3742 llvm::Value *Args[] = {
3743 emitUpdateLocation(CGF, Loc,
3744 isOpenMPDistributeDirective(DKind)
3745 ? OMP_IDENT_WORK_DISTRIBUTE
3746 : isOpenMPLoopDirective(DKind)
3747 ? OMP_IDENT_WORK_LOOP
3748 : OMP_IDENT_WORK_SECTIONS),
3749 getThreadID(CGF, Loc)};
3750 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3751 Args);
3752}
3753
3754void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3755 SourceLocation Loc,
3756 unsigned IVSize,
3757 bool IVSigned) {
3758 if (!CGF.HaveInsertPoint())
3759 return;
3760 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3761 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3762 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3763}
3764
3765llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3766 SourceLocation Loc, unsigned IVSize,
3767 bool IVSigned, Address IL,
3768 Address LB, Address UB,
3769 Address ST) {
3770 // Call __kmpc_dispatch_next(
3771 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3772 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3773 // kmp_int[32|64] *p_stride);
3774 llvm::Value *Args[] = {
3775 emitUpdateLocation(CGF, Loc),
3776 getThreadID(CGF, Loc),
3777 IL.getPointer(), // &isLastIter
3778 LB.getPointer(), // &Lower
3779 UB.getPointer(), // &Upper
3780 ST.getPointer() // &Stride
3781 };
3782 llvm::Value *Call =
3783 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3784 return CGF.EmitScalarConversion(
3785 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3786 CGF.getContext().BoolTy, Loc);
3787}
3788
3789void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3790 llvm::Value *NumThreads,
3791 SourceLocation Loc) {
3792 if (!CGF.HaveInsertPoint())
3793 return;
3794 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3795 llvm::Value *Args[] = {
3796 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3797 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3798 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3799 Args);
3800}
3801
3802void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3803 OpenMPProcBindClauseKind ProcBind,
3804 SourceLocation Loc) {
3805 if (!CGF.HaveInsertPoint())
3806 return;
3807 // Constants for proc bind value accepted by the runtime.
3808 enum ProcBindTy {
3809 ProcBindFalse = 0,
3810 ProcBindTrue,
3811 ProcBindMaster,
3812 ProcBindClose,
3813 ProcBindSpread,
3814 ProcBindIntel,
3815 ProcBindDefault
3816 } RuntimeProcBind;
3817 switch (ProcBind) {
3818 case OMPC_PROC_BIND_master:
3819 RuntimeProcBind = ProcBindMaster;
3820 break;
3821 case OMPC_PROC_BIND_close:
3822 RuntimeProcBind = ProcBindClose;
3823 break;
3824 case OMPC_PROC_BIND_spread:
3825 RuntimeProcBind = ProcBindSpread;
3826 break;
3827 case OMPC_PROC_BIND_unknown:
3828 llvm_unreachable("Unsupported proc_bind value.")::llvm::llvm_unreachable_internal("Unsupported proc_bind value."
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3828)
;
3829 }
3830 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3831 llvm::Value *Args[] = {
3832 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3833 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3834 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3835}
3836
3837void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3838 SourceLocation Loc) {
3839 if (!CGF.HaveInsertPoint())
3840 return;
3841 // Build call void __kmpc_flush(ident_t *loc)
3842 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3843 emitUpdateLocation(CGF, Loc));
3844}
3845
3846namespace {
3847/// Indexes of fields for type kmp_task_t.
3848enum KmpTaskTFields {
3849 /// List of shared variables.
3850 KmpTaskTShareds,
3851 /// Task routine.
3852 KmpTaskTRoutine,
3853 /// Partition id for the untied tasks.
3854 KmpTaskTPartId,
3855 /// Function with call of destructors for private variables.
3856 Data1,
3857 /// Task priority.
3858 Data2,
3859 /// (Taskloops only) Lower bound.
3860 KmpTaskTLowerBound,
3861 /// (Taskloops only) Upper bound.
3862 KmpTaskTUpperBound,
3863 /// (Taskloops only) Stride.
3864 KmpTaskTStride,
3865 /// (Taskloops only) Is last iteration flag.
3866 KmpTaskTLastIter,
3867 /// (Taskloops only) Reduction data.
3868 KmpTaskTReductions,
3869};
3870} // anonymous namespace
3871
3872bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3873 return OffloadEntriesTargetRegion.empty() &&
3874 OffloadEntriesDeviceGlobalVar.empty();
3875}
3876
3877/// Initialize target region entry.
3878void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3879 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3880 StringRef ParentName, unsigned LineNum,
3881 unsigned Order) {
3882 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3884, __PRETTY_FUNCTION__))
3883 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3884, __PRETTY_FUNCTION__))
3884 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3884, __PRETTY_FUNCTION__))
;
3885 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3886 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3887 OMPTargetRegionEntryTargetRegion);
3888 ++OffloadingEntriesNum;
3889}
3890
3891void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3892 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3893 StringRef ParentName, unsigned LineNum,
3894 llvm::Constant *Addr, llvm::Constant *ID,
3895 OMPTargetRegionEntryKind Flags) {
3896 // If we are emitting code for a target, the entry is already initialized,
3897 // only has to be registered.
3898 if (CGM.getLangOpts().OpenMPIsDevice) {
3899 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3900 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3901 DiagnosticsEngine::Error,
3902 "Unable to find target region on line '%0' in the device code.");
3903 CGM.getDiags().Report(DiagID) << LineNum;
3904 return;
3905 }
3906 auto &Entry =
3907 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3908 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3908, __PRETTY_FUNCTION__))
;
3909 Entry.setAddress(Addr);
3910 Entry.setID(ID);
3911 Entry.setFlags(Flags);
3912 } else {
3913 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3914 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3915 ++OffloadingEntriesNum;
3916 }
3917}
3918
3919bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3920 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3921 unsigned LineNum) const {
3922 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3923 if (PerDevice == OffloadEntriesTargetRegion.end())
3924 return false;
3925 auto PerFile = PerDevice->second.find(FileID);
3926 if (PerFile == PerDevice->second.end())
3927 return false;
3928 auto PerParentName = PerFile->second.find(ParentName);
3929 if (PerParentName == PerFile->second.end())
3930 return false;
3931 auto PerLine = PerParentName->second.find(LineNum);
3932 if (PerLine == PerParentName->second.end())
3933 return false;
3934 // Fail if this entry is already registered.
3935 if (PerLine->second.getAddress() || PerLine->second.getID())
3936 return false;
3937 return true;
3938}
3939
3940void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3941 const OffloadTargetRegionEntryInfoActTy &Action) {
3942 // Scan all target region entries and perform the provided action.
3943 for (const auto &D : OffloadEntriesTargetRegion)
3944 for (const auto &F : D.second)
3945 for (const auto &P : F.second)
3946 for (const auto &L : P.second)
3947 Action(D.first, F.first, P.first(), L.first, L.second);
3948}
3949
3950void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3951 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3952 OMPTargetGlobalVarEntryKind Flags,
3953 unsigned Order) {
3954 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3956, __PRETTY_FUNCTION__))
3955 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3956, __PRETTY_FUNCTION__))
3956 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3956, __PRETTY_FUNCTION__))
;
3957 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3958 ++OffloadingEntriesNum;
3959}
3960
3961void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3962 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3963 CharUnits VarSize,
3964 OMPTargetGlobalVarEntryKind Flags,
3965 llvm::GlobalValue::LinkageTypes Linkage) {
3966 if (CGM.getLangOpts().OpenMPIsDevice) {
3967 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3968 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
3969 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
;
3970 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3971, __PRETTY_FUNCTION__))
3971 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3971, __PRETTY_FUNCTION__))
;
3972 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3973 if (Entry.getVarSize().isZero()) {
3974 Entry.setVarSize(VarSize);
3975 Entry.setLinkage(Linkage);
3976 }
3977 return;
3978 }
3979 Entry.setVarSize(VarSize);
3980 Entry.setLinkage(Linkage);
3981 Entry.setAddress(Addr);
3982 } else {
3983 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3984 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3985 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
3986 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
;
3987 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3988, __PRETTY_FUNCTION__))
3988 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3988, __PRETTY_FUNCTION__))
;
3989 if (Entry.getVarSize().isZero()) {
3990 Entry.setVarSize(VarSize);
3991 Entry.setLinkage(Linkage);
3992 }
3993 return;
3994 }
3995 OffloadEntriesDeviceGlobalVar.try_emplace(
3996 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3997 ++OffloadingEntriesNum;
3998 }
3999}
4000
4001void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4002 actOnDeviceGlobalVarEntriesInfo(
4003 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4004 // Scan all target region entries and perform the provided action.
4005 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4006 Action(E.getKey(), E.getValue());
4007}
4008
4009llvm::Function *
4010CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
4011 // If we don't have entries or if we are emitting code for the device, we
4012 // don't need to do anything.
4013 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
4014 return nullptr;
4015
4016 llvm::Module &M = CGM.getModule();
4017 ASTContext &C = CGM.getContext();
4018
4019 // Get list of devices we care about
4020 const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
4021
4022 // We should be creating an offloading descriptor only if there are devices
4023 // specified.
4024 assert(!Devices.empty() && "No OpenMP offloading devices??")((!Devices.empty() && "No OpenMP offloading devices??"
) ? static_cast<void> (0) : __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4024, __PRETTY_FUNCTION__))
;
4025
4026 // Create the external variables that will point to the begin and end of the
4027 // host entries section. These will be defined by the linker.
4028 llvm::Type *OffloadEntryTy =
4029 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
4030 auto *HostEntriesBegin = new llvm::GlobalVariable(
4031 M, OffloadEntryTy, /*isConstant=*/true,
4032 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
4033 "__start_omp_offloading_entries");
4034 HostEntriesBegin->setVisibility(llvm::GlobalValue::HiddenVisibility);
4035 auto *HostEntriesEnd = new llvm::GlobalVariable(
4036 M, OffloadEntryTy, /*isConstant=*/true,
4037 llvm::GlobalValue::ExternalLinkage,
4038 /*Initializer=*/nullptr, "__stop_omp_offloading_entries");
4039 HostEntriesEnd->setVisibility(llvm::GlobalValue::HiddenVisibility);
4040
4041 // Create all device images
4042 auto *DeviceImageTy = cast<llvm::StructType>(
4043 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
4044 ConstantInitBuilder DeviceImagesBuilder(CGM);
4045 ConstantArrayBuilder DeviceImagesEntries =
4046 DeviceImagesBuilder.beginArray(DeviceImageTy);
4047
4048 for (const llvm::Triple &Device : Devices) {
4049 StringRef T = Device.getTriple();
4050 std::string BeginName = getName({"omp_offloading", "img_start", ""});
4051 auto *ImgBegin = new llvm::GlobalVariable(
4052 M, CGM.Int8Ty, /*isConstant=*/true,
4053 llvm::GlobalValue::ExternalWeakLinkage,
4054 /*Initializer=*/nullptr, Twine(BeginName).concat(T));
4055 std::string EndName = getName({"omp_offloading", "img_end", ""});
4056 auto *ImgEnd = new llvm::GlobalVariable(
4057 M, CGM.Int8Ty, /*isConstant=*/true,
4058 llvm::GlobalValue::ExternalWeakLinkage,
4059 /*Initializer=*/nullptr, Twine(EndName).concat(T));
4060
4061 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
4062 HostEntriesEnd};
4063 createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
4064 DeviceImagesEntries);
4065 }
4066
4067 // Create device images global array.
4068 std::string ImagesName = getName({"omp_offloading", "device_images"});
4069 llvm::GlobalVariable *DeviceImages =
4070 DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
4071 CGM.getPointerAlign(),
4072 /*isConstant=*/true);
4073 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4074
4075 // This is a Zero array to be used in the creation of the constant expressions
4076 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
4077 llvm::Constant::getNullValue(CGM.Int32Ty)};
4078
4079 // Create the target region descriptor.
4080 llvm::Constant *Data[] = {
4081 llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
4082 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
4083 DeviceImages, Index),
4084 HostEntriesBegin, HostEntriesEnd};
4085 std::string Descriptor = getName({"omp_offloading", "descriptor"});
4086 llvm::GlobalVariable *Desc = createGlobalStruct(
4087 CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
4088
4089 // Emit code to register or unregister the descriptor at execution
4090 // startup or closing, respectively.
4091
4092 llvm::Function *UnRegFn;
4093 {
4094 FunctionArgList Args;
4095 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
4096 Args.push_back(&DummyPtr);
4097
4098 CodeGenFunction CGF(CGM);
4099 // Disable debug info for global (de-)initializer because they are not part
4100 // of some particular construct.
4101 CGF.disableDebugInfo();
4102 const auto &FI =
4103 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4104 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4105 std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
4106 UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
4107 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
4108 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
4109 Desc);
4110 CGF.FinishFunction();
4111 }
4112 llvm::Function *RegFn;
4113 {
4114 CodeGenFunction CGF(CGM);
4115 // Disable debug info for global (de-)initializer because they are not part
4116 // of some particular construct.
4117 CGF.disableDebugInfo();
4118 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
4119 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4120
4121 // Encode offload target triples into the registration function name. It
4122 // will serve as a comdat key for the registration/unregistration code for
4123 // this particular combination of offloading targets.
4124 SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
4125 RegFnNameParts[0] = "omp_offloading";
4126 RegFnNameParts[1] = "descriptor_reg";
4127 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
4128 [](const llvm::Triple &T) -> const std::string& {
4129 return T.getTriple();
4130 });
4131 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
4132 std::string Descriptor = getName(RegFnNameParts);
4133 RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
4134 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
4135 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
4136 // Create a variable to drive the registration and unregistration of the
4137 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
4138 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
4139 SourceLocation(), nullptr, C.CharTy,
4140 ImplicitParamDecl::Other);
4141 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
4142 CGF.FinishFunction();
4143 }
4144 if (CGM.supportsCOMDAT()) {
4145 // It is sufficient to call registration function only once, so create a
4146 // COMDAT group for registration/unregistration functions and associated
4147 // data. That would reduce startup time and code size. Registration
4148 // function serves as a COMDAT group key.
4149 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
4150 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
4151 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
4152 RegFn->setComdat(ComdatKey);
4153 UnRegFn->setComdat(ComdatKey);
4154 DeviceImages->setComdat(ComdatKey);
4155 Desc->setComdat(ComdatKey);
4156 }
4157 return RegFn;
4158}
4159
4160void CGOpenMPRuntime::createOffloadEntry(
4161 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4162 llvm::GlobalValue::LinkageTypes Linkage) {
4163 StringRef Name = Addr->getName();
4164 llvm::Module &M = CGM.getModule();
4165 llvm::LLVMContext &C = M.getContext();
4166
4167 // Create constant string with the name.
4168 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4169
4170 std::string StringName = getName({"omp_offloading", "entry_name"});
4171 auto *Str = new llvm::GlobalVariable(
4172 M, StrPtrInit->getType(), /*isConstant=*/true,
4173 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4174 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4175
4176 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4177 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4178 llvm::ConstantInt::get(CGM.SizeTy, Size),
4179 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4180 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4181 std::string EntryName = getName({"omp_offloading", "entry", ""});
4182 llvm::GlobalVariable *Entry = createGlobalStruct(
4183 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4184 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4185
4186 // The entry has to be created in the section the linker expects it to be.
4187 Entry->setSection("omp_offloading_entries");
4188}
4189
4190void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4191 // Emit the offloading entries and metadata so that the device codegen side
4192 // can easily figure out what to emit. The produced metadata looks like
4193 // this:
4194 //
4195 // !omp_offload.info = !{!1, ...}
4196 //
4197 // Right now we only generate metadata for function that contain target
4198 // regions.
4199
4200 // If we do not have entries, we don't need to do anything.
4201 if (OffloadEntriesInfoManager.empty())
4202 return;
4203
4204 llvm::Module &M = CGM.getModule();
4205 llvm::LLVMContext &C = M.getContext();
4206 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4207 SourceLocation, StringRef>,
4208 16>
4209 OrderedEntries(OffloadEntriesInfoManager.size());
4210 llvm::SmallVector<StringRef, 16> ParentFunctions(
4211 OffloadEntriesInfoManager.size());
4212
4213 // Auxiliary methods to create metadata values and strings.
4214 auto &&GetMDInt = [this](unsigned V) {
4215 return llvm::ConstantAsMetadata::get(
4216 llvm::ConstantInt::get(CGM.Int32Ty, V));
4217 };
4218
4219 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4220
4221 // Create the offloading info metadata node.
4222 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4223
4224 // Create function that emits metadata for each target region entry;
4225 auto &&TargetRegionMetadataEmitter =
4226 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4227 &GetMDString](
4228 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4229 unsigned Line,
4230 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4231 // Generate metadata for target regions. Each entry of this metadata
4232 // contains:
4233 // - Entry 0 -> Kind of this type of metadata (0).
4234 // - Entry 1 -> Device ID of the file where the entry was identified.
4235 // - Entry 2 -> File ID of the file where the entry was identified.
4236 // - Entry 3 -> Mangled name of the function where the entry was
4237 // identified.
4238 // - Entry 4 -> Line in the file where the entry was identified.
4239 // - Entry 5 -> Order the entry was created.
4240 // The first element of the metadata node is the kind.
4241 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4242 GetMDInt(FileID), GetMDString(ParentName),
4243 GetMDInt(Line), GetMDInt(E.getOrder())};
4244
4245 SourceLocation Loc;
4246 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4247 E = CGM.getContext().getSourceManager().fileinfo_end();
4248 I != E; ++I) {
4249 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4250 I->getFirst()->getUniqueID().getFile() == FileID) {
4251 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4252 I->getFirst(), Line, 1);
4253 break;
4254 }
4255 }
4256 // Save this entry in the right position of the ordered entries array.
4257 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4258 ParentFunctions[E.getOrder()] = ParentName;
4259
4260 // Add metadata to the named metadata node.
4261 MD->addOperand(llvm::MDNode::get(C, Ops));
4262 };
4263
4264 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4265 TargetRegionMetadataEmitter);
4266
4267 // Create function that emits metadata for each device global variable entry;
4268 auto &&DeviceGlobalVarMetadataEmitter =
4269 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4270 MD](StringRef MangledName,
4271 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4272 &E) {
4273 // Generate metadata for global variables. Each entry of this metadata
4274 // contains:
4275 // - Entry 0 -> Kind of this type of metadata (1).
4276 // - Entry 1 -> Mangled name of the variable.
4277 // - Entry 2 -> Declare target kind.
4278 // - Entry 3 -> Order the entry was created.
4279 // The first element of the metadata node is the kind.
4280 llvm::Metadata *Ops[] = {
4281 GetMDInt(E.getKind()), GetMDString(MangledName),
4282 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4283
4284 // Save this entry in the right position of the ordered entries array.
4285 OrderedEntries[E.getOrder()] =
4286 std::make_tuple(&E, SourceLocation(), MangledName);
4287
4288 // Add metadata to the named metadata node.
4289 MD->addOperand(llvm::MDNode::get(C, Ops));
4290 };
4291
4292 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4293 DeviceGlobalVarMetadataEmitter);
4294
4295 for (const auto &E : OrderedEntries) {
4296 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4296, __PRETTY_FUNCTION__))
;
4297 if (const auto *CE =
4298 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4299 std::get<0>(E))) {
4300 if (!CE->getID() || !CE->getAddress()) {
4301 // Do not blame the entry if the parent funtion is not emitted.
4302 StringRef FnName = ParentFunctions[CE->getOrder()];
4303 if (!CGM.GetGlobalValue(FnName))
4304 continue;
4305 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4306 DiagnosticsEngine::Error,
4307 "Offloading entry for target region in %0 is incorrect: either the "
4308 "address or the ID is invalid.");
4309 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4310 continue;
4311 }
4312 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4313 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4314 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
4315 OffloadEntryInfoDeviceGlobalVar>(
4316 std::get<0>(E))) {
4317 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4318 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4319 CE->getFlags());
4320 switch (Flags) {
4321 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4322 if (CGM.getLangOpts().OpenMPIsDevice &&
4323 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4324 continue;
4325 if (!CE->getAddress()) {
4326 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4327 DiagnosticsEngine::Error, "Offloading entry for declare target "
4328 "variable %0 is incorrect: the "
4329 "address is invalid.");
4330 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4331 continue;
4332 }
4333 // The vaiable has no definition - no need to add the entry.
4334 if (CE->getVarSize().isZero())
4335 continue;
4336 break;
4337 }
4338 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4339 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4341, __PRETTY_FUNCTION__))
4340 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4341, __PRETTY_FUNCTION__))
4341 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4341, __PRETTY_FUNCTION__))
;
4342 if (CGM.getLangOpts().OpenMPIsDevice)
4343 continue;
4344 if (!CE->getAddress()) {
4345 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4346 DiagnosticsEngine::Error,
4347 "Offloading entry for declare target variable is incorrect: the "
4348 "address is invalid.");
4349 CGM.getDiags().Report(DiagID);
4350 continue;
4351 }
4352 break;
4353 }
4354 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4355 CE->getVarSize().getQuantity(), Flags,
4356 CE->getLinkage());
4357 } else {
4358 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4358)
;
4359 }
4360 }
4361}
4362
4363/// Loads all the offload entries information from the host IR
4364/// metadata.
4365void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4366 // If we are in target mode, load the metadata from the host IR. This code has
4367 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4368
4369 if (!CGM.getLangOpts().OpenMPIsDevice)
4370 return;
4371
4372 if (CGM.getLangOpts().OMPHostIRFile.empty())
4373 return;
4374
4375 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4376 if (auto EC = Buf.getError()) {
4377 CGM.getDiags().Report(diag::err_cannot_open_file)
4378 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4379 return;
4380 }
4381
4382 llvm::LLVMContext C;
4383 auto ME = expectedToErrorOrAndEmitErrors(
4384 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4385
4386 if (auto EC = ME.getError()) {
4387 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4388 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4389 CGM.getDiags().Report(DiagID)
4390 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4391 return;
4392 }
4393
4394 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4395 if (!MD)
4396 return;
4397
4398 for (llvm::MDNode *MN : MD->operands()) {
4399 auto &&GetMDInt = [MN](unsigned Idx) {
4400 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4401 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4402 };
4403
4404 auto &&GetMDString = [MN](unsigned Idx) {
4405 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4406 return V->getString();
4407 };
4408
4409 switch (GetMDInt(0)) {
4410 default:
4411 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4411)
;
4412 break;
4413 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4414 OffloadingEntryInfoTargetRegion:
4415 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4416 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4417 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4418 /*Order=*/GetMDInt(5));
4419 break;
4420 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4421 OffloadingEntryInfoDeviceGlobalVar:
4422 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4423 /*MangledName=*/GetMDString(1),
4424 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4425 /*Flags=*/GetMDInt(2)),
4426 /*Order=*/GetMDInt(3));
4427 break;
4428 }
4429 }
4430}
4431
4432void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4433 if (!KmpRoutineEntryPtrTy) {
4434 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4435 ASTContext &C = CGM.getContext();
4436 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4437 FunctionProtoType::ExtProtoInfo EPI;
4438 KmpRoutineEntryPtrQTy = C.getPointerType(
4439 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4440 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4441 }
4442}
4443
4444QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4445 // Make sure the type of the entry is already created. This is the type we
4446 // have to create:
4447 // struct __tgt_offload_entry{
4448 // void *addr; // Pointer to the offload entry info.
4449 // // (function or global)
4450 // char *name; // Name of the function or global.
4451 // size_t size; // Size of the entry info (0 if it a function).
4452 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4453 // int32_t reserved; // Reserved, to use by the runtime library.
4454 // };
4455 if (TgtOffloadEntryQTy.isNull()) {
4456 ASTContext &C = CGM.getContext();
4457 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4458 RD->startDefinition();
4459 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4460 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4461 addFieldToRecordDecl(C, RD, C.getSizeType());
4462 addFieldToRecordDecl(
4463 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4464 addFieldToRecordDecl(
4465 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4466 RD->completeDefinition();
4467 RD->addAttr(PackedAttr::CreateImplicit(C));
4468 TgtOffloadEntryQTy = C.getRecordType(RD);
4469 }
4470 return TgtOffloadEntryQTy;
4471}
4472
4473QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4474 // These are the types we need to build:
4475 // struct __tgt_device_image{
4476 // void *ImageStart; // Pointer to the target code start.
4477 // void *ImageEnd; // Pointer to the target code end.
4478 // // We also add the host entries to the device image, as it may be useful
4479 // // for the target runtime to have access to that information.
4480 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
4481 // // the entries.
4482 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4483 // // entries (non inclusive).
4484 // };
4485 if (TgtDeviceImageQTy.isNull()) {
4486 ASTContext &C = CGM.getContext();
4487 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4488 RD->startDefinition();
4489 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4490 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4491 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4492 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4493 RD->completeDefinition();
4494 TgtDeviceImageQTy = C.getRecordType(RD);
4495 }
4496 return TgtDeviceImageQTy;
4497}
4498
4499QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4500 // struct __tgt_bin_desc{
4501 // int32_t NumDevices; // Number of devices supported.
4502 // __tgt_device_image *DeviceImages; // Arrays of device images
4503 // // (one per device).
4504 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
4505 // // entries.
4506 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4507 // // entries (non inclusive).
4508 // };
4509 if (TgtBinaryDescriptorQTy.isNull()) {
4510 ASTContext &C = CGM.getContext();
4511 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4512 RD->startDefinition();
4513 addFieldToRecordDecl(
4514 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4515 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4516 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4517 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4518 RD->completeDefinition();
4519 TgtBinaryDescriptorQTy = C.getRecordType(RD);
4520 }
4521 return TgtBinaryDescriptorQTy;
4522}
4523
4524namespace {
4525struct PrivateHelpersTy {
4526 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4527 const VarDecl *PrivateElemInit)
4528 : Original(Original), PrivateCopy(PrivateCopy),
4529 PrivateElemInit(PrivateElemInit) {}
4530 const VarDecl *Original;
4531 const VarDecl *PrivateCopy;
4532 const VarDecl *PrivateElemInit;
4533};
4534typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4535} // anonymous namespace
4536
4537static RecordDecl *
4538createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4539 if (!Privates.empty()) {
4540 ASTContext &C = CGM.getContext();
4541 // Build struct .kmp_privates_t. {
4542 // /* private vars */
4543 // };
4544 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4545 RD->startDefinition();
4546 for (const auto &Pair : Privates) {
4547 const VarDecl *VD = Pair.second.Original;
4548 QualType Type = VD->getType().getNonReferenceType();
4549 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4550 if (VD->hasAttrs()) {
4551 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4552 E(VD->getAttrs().end());
4553 I != E; ++I)
4554 FD->addAttr(*I);
4555 }
4556 }
4557 RD->completeDefinition();
4558 return RD;
4559 }
4560 return nullptr;
4561}
4562
4563static RecordDecl *
4564createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4565 QualType KmpInt32Ty,
4566 QualType KmpRoutineEntryPointerQTy) {
4567 ASTContext &C = CGM.getContext();
4568 // Build struct kmp_task_t {
4569 // void * shareds;
4570 // kmp_routine_entry_t routine;
4571 // kmp_int32 part_id;
4572 // kmp_cmplrdata_t data1;
4573 // kmp_cmplrdata_t data2;
4574 // For taskloops additional fields:
4575 // kmp_uint64 lb;
4576 // kmp_uint64 ub;
4577 // kmp_int64 st;
4578 // kmp_int32 liter;
4579 // void * reductions;
4580 // };
4581 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4582 UD->startDefinition();
4583 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4584 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4585 UD->completeDefinition();
4586 QualType KmpCmplrdataTy = C.getRecordType(UD);
4587 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4588 RD->startDefinition();
4589 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4590 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4591 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4592 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4593 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4594 if (isOpenMPTaskLoopDirective(Kind)) {
4595 QualType KmpUInt64Ty =
4596 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4597 QualType KmpInt64Ty =
4598 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4599 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4600 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4601 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4602 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4603 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4604 }
4605 RD->completeDefinition();
4606 return RD;
4607}
4608
4609static RecordDecl *
4610createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4611 ArrayRef<PrivateDataTy> Privates) {
4612 ASTContext &C = CGM.getContext();
4613 // Build struct kmp_task_t_with_privates {
4614 // kmp_task_t task_data;
4615 // .kmp_privates_t. privates;
4616 // };
4617 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4618 RD->startDefinition();
4619 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4620 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4621 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4622 RD->completeDefinition();
4623 return RD;
4624}
4625
4626/// Emit a proxy function which accepts kmp_task_t as the second
4627/// argument.
4628/// \code
4629/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4630/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4631/// For taskloops:
4632/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4633/// tt->reductions, tt->shareds);
4634/// return 0;
4635/// }
4636/// \endcode
4637static llvm::Function *
4638emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4639 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4640 QualType KmpTaskTWithPrivatesPtrQTy,
4641 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4642 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4643 llvm::Value *TaskPrivatesMap) {
4644 ASTContext &C = CGM.getContext();
4645 FunctionArgList Args;
4646 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4647 ImplicitParamDecl::Other);
4648 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4649 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4650 ImplicitParamDecl::Other);
4651 Args.push_back(&GtidArg);
4652 Args.push_back(&TaskTypeArg);
4653 const auto &TaskEntryFnInfo =
4654 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4655 llvm::FunctionType *TaskEntryTy =
4656 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4657 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4658 auto *TaskEntry = llvm::Function::Create(
4659 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4660 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4661 TaskEntry->setDoesNotRecurse();
4662 CodeGenFunction CGF(CGM);
4663 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4664 Loc, Loc);
4665
4666 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4667 // tt,
4668 // For taskloops:
4669 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4670 // tt->task_data.shareds);
4671 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4672 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4673 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4674 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4675 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4676 const auto *KmpTaskTWithPrivatesQTyRD =
4677 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4678 LValue Base =
4679 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4680 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4681 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4682 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4683 llvm::Value *PartidParam = PartIdLVal.getPointer();
4684
4685 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4686 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4687 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4688 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4689 CGF.ConvertTypeForMem(SharedsPtrTy));
4690
4691 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4692 llvm::Value *PrivatesParam;
4693 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4694 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4695 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4696 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4697 } else {
4698 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4699 }
4700
4701 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4702 TaskPrivatesMap,
4703 CGF.Builder
4704 .CreatePointerBitCastOrAddrSpaceCast(
4705 TDBase.getAddress(), CGF.VoidPtrTy)
4706 .getPointer()};
4707 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4708 std::end(CommonArgs));
4709 if (isOpenMPTaskLoopDirective(Kind)) {
4710 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4711 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4712 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4713 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4714 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4715 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4716 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4717 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4718 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4719 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4720 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4721 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4722 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4723 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4724 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4725 CallArgs.push_back(LBParam);
4726 CallArgs.push_back(UBParam);
4727 CallArgs.push_back(StParam);
4728 CallArgs.push_back(LIParam);
4729 CallArgs.push_back(RParam);
4730 }
4731 CallArgs.push_back(SharedsParam);
4732
4733 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4734 CallArgs);
4735 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4736 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4737 CGF.FinishFunction();
4738 return TaskEntry;
4739}
4740
4741static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4742 SourceLocation Loc,
4743 QualType KmpInt32Ty,
4744 QualType KmpTaskTWithPrivatesPtrQTy,
4745 QualType KmpTaskTWithPrivatesQTy) {
4746 ASTContext &C = CGM.getContext();
4747 FunctionArgList Args;
4748 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4749 ImplicitParamDecl::Other);
4750 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4751 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4752 ImplicitParamDecl::Other);
4753 Args.push_back(&GtidArg);
4754 Args.push_back(&TaskTypeArg);
4755 const auto &DestructorFnInfo =
4756 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4757 llvm::FunctionType *DestructorFnTy =
4758 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4759 std::string Name =
4760 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4761 auto *DestructorFn =
4762 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4763 Name, &CGM.getModule());
4764 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4765 DestructorFnInfo);
4766 DestructorFn->setDoesNotRecurse();
4767 CodeGenFunction CGF(CGM);
4768 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4769 Args, Loc, Loc);
4770
4771 LValue Base = CGF.EmitLoadOfPointerLValue(
4772 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4773 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4774 const auto *KmpTaskTWithPrivatesQTyRD =
4775 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4776 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4777 Base = CGF.EmitLValueForField(Base, *FI);
4778 for (const auto *Field :
4779 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4780 if (QualType::DestructionKind DtorKind =
4781 Field->getType().isDestructedType()) {
4782 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4783 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4784 }
4785 }
4786 CGF.FinishFunction();
4787 return DestructorFn;
4788}
4789
4790/// Emit a privates mapping function for correct handling of private and
4791/// firstprivate variables.
4792/// \code
4793/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4794/// **noalias priv1,..., <tyn> **noalias privn) {
4795/// *priv1 = &.privates.priv1;
4796/// ...;
4797/// *privn = &.privates.privn;
4798/// }
4799/// \endcode
4800static llvm::Value *
4801emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4802 ArrayRef<const Expr *> PrivateVars,
4803 ArrayRef<const Expr *> FirstprivateVars,
4804 ArrayRef<const Expr *> LastprivateVars,
4805 QualType PrivatesQTy,
4806 ArrayRef<PrivateDataTy> Privates) {
4807 ASTContext &C = CGM.getContext();
4808 FunctionArgList Args;
4809 ImplicitParamDecl TaskPrivatesArg(
4810 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4811 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4812 ImplicitParamDecl::Other);
4813 Args.push_back(&TaskPrivatesArg);
4814 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4815 unsigned Counter = 1;
4816 for (const Expr *E : PrivateVars) {
4817 Args.push_back(ImplicitParamDecl::Create(
4818 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4819 C.getPointerType(C.getPointerType(E->getType()))
4820 .withConst()
4821 .withRestrict(),
4822 ImplicitParamDecl::Other));
4823 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4824 PrivateVarsPos[VD] = Counter;
4825 ++Counter;
4826 }
4827 for (const Expr *E : FirstprivateVars) {
4828 Args.push_back(ImplicitParamDecl::Create(
4829 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4830 C.getPointerType(C.getPointerType(E->getType()))
4831 .withConst()
4832 .withRestrict(),
4833 ImplicitParamDecl::Other));
4834 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4835 PrivateVarsPos[VD] = Counter;
4836 ++Counter;
4837 }
4838 for (const Expr *E : LastprivateVars) {
4839 Args.push_back(ImplicitParamDecl::Create(
4840 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4841 C.getPointerType(C.getPointerType(E->getType()))
4842 .withConst()
4843 .withRestrict(),
4844 ImplicitParamDecl::Other));
4845 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4846 PrivateVarsPos[VD] = Counter;
4847 ++Counter;
4848 }
4849 const auto &TaskPrivatesMapFnInfo =
4850 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4851 llvm::FunctionType *TaskPrivatesMapTy =
4852 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4853 std::string Name =
4854 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4855 auto *TaskPrivatesMap = llvm::Function::Create(
4856 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4857 &CGM.getModule());
4858 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4859 TaskPrivatesMapFnInfo);
4860 if (CGM.getLangOpts().Optimize) {
4861 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4862 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4863 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4864 }
4865 CodeGenFunction CGF(CGM);
4866 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4867 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4868
4869 // *privi = &.privates.privi;
4870 LValue Base = CGF.EmitLoadOfPointerLValue(
4871 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4872 TaskPrivatesArg.getType()->castAs<PointerType>());
4873 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4874 Counter = 0;
4875 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4876 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4877 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4878 LValue RefLVal =
4879 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4880 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4881 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4882 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4883 ++Counter;
4884 }
4885 CGF.FinishFunction();
4886 return TaskPrivatesMap;
4887}
4888
4889/// Emit initialization for private variables in task-based directives.
4890static void emitPrivatesInit(CodeGenFunction &CGF,
4891 const OMPExecutableDirective &D,
4892 Address KmpTaskSharedsPtr, LValue TDBase,
4893 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4894 QualType SharedsTy, QualType SharedsPtrTy,
4895 const OMPTaskDataTy &Data,
4896 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4897 ASTContext &C = CGF.getContext();
4898 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4899 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4900 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4901 ? OMPD_taskloop
4902 : OMPD_task;
4903 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4904 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4905 LValue SrcBase;
4906 bool IsTargetTask =
4907 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4908 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4909 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4910 // PointersArray and SizesArray. The original variables for these arrays are
4911 // not captured and we get their addresses explicitly.
4912 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4913 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4914 SrcBase = CGF.MakeAddrLValue(
4915 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4916 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4917 SharedsTy);
4918 }
4919 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4920 for (const PrivateDataTy &Pair : Privates) {
4921 const VarDecl *VD = Pair.second.PrivateCopy;
4922 const Expr *Init = VD->getAnyInitializer();
4923 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4924 !CGF.isTrivialInitializer(Init)))) {
4925 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4926 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4927 const VarDecl *OriginalVD = Pair.second.Original;
4928 // Check if the variable is the target-based BasePointersArray,
4929 // PointersArray or SizesArray.
4930 LValue SharedRefLValue;
4931 QualType Type = PrivateLValue.getType();
4932 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4933 if (IsTargetTask && !SharedField) {
4934 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4935 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4936 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4937 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4938 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4939 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4940 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
4941 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4941, __PRETTY_FUNCTION__))
;
4942 SharedRefLValue =
4943 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4944 } else {
4945 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4946 SharedRefLValue = CGF.MakeAddrLValue(
4947 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4948 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4949 SharedRefLValue.getTBAAInfo());
4950 }
4951 if (Type->isArrayType()) {
4952 // Initialize firstprivate array.
4953 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4954 // Perform simple memcpy.
4955 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4956 } else {
4957 // Initialize firstprivate array using element-by-element
4958 // initialization.
4959 CGF.EmitOMPAggregateAssign(
4960 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4961 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4962 Address SrcElement) {
4963 // Clean up any temporaries needed by the initialization.
4964 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4965 InitScope.addPrivate(
4966 Elem, [SrcElement]() -> Address { return SrcElement; });
4967 (void)InitScope.Privatize();
4968 // Emit initialization for single element.
4969 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4970 CGF, &CapturesInfo);
4971 CGF.EmitAnyExprToMem(Init, DestElement,
4972 Init->getType().getQualifiers(),
4973 /*IsInitializer=*/false);
4974 });
4975 }
4976 } else {
4977 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4978 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4979 return SharedRefLValue.getAddress();
4980 });
4981 (void)InitScope.Privatize();
4982 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4983 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4984 /*capturedByInit=*/false);
4985 }
4986 } else {
4987 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4988 }
4989 }
4990 ++FI;
4991 }
4992}
4993
4994/// Check if duplication function is required for taskloops.
4995static bool checkInitIsRequired(CodeGenFunction &CGF,
4996 ArrayRef<PrivateDataTy> Privates) {
4997 bool InitRequired = false;
4998 for (const PrivateDataTy &Pair : Privates) {
4999 const VarDecl *VD = Pair.second.PrivateCopy;
5000 const Expr *Init = VD->getAnyInitializer();
5001 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
5002 !CGF.isTrivialInitializer(Init));
5003 if (InitRequired)
5004 break;
5005 }
5006 return InitRequired;
5007}
5008
5009
5010/// Emit task_dup function (for initialization of
5011/// private/firstprivate/lastprivate vars and last_iter flag)
5012/// \code
5013/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
5014/// lastpriv) {
5015/// // setup lastprivate flag
5016/// task_dst->last = lastpriv;
5017/// // could be constructor calls here...
5018/// }
5019/// \endcode
5020static llvm::Value *
5021emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
5022 const OMPExecutableDirective &D,
5023 QualType KmpTaskTWithPrivatesPtrQTy,
5024 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
5025 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
5026 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
5027 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
5028 ASTContext &C = CGM.getContext();
5029 FunctionArgList Args;
5030 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5031 KmpTaskTWithPrivatesPtrQTy,
5032 ImplicitParamDecl::Other);
5033 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5034 KmpTaskTWithPrivatesPtrQTy,
5035 ImplicitParamDecl::Other);
5036 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
5037 ImplicitParamDecl::Other);
5038 Args.push_back(&DstArg);
5039 Args.push_back(&SrcArg);
5040 Args.push_back(&LastprivArg);
5041 const auto &TaskDupFnInfo =
5042 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5043 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
5044 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
5045 auto *TaskDup = llvm::Function::Create(
5046 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
5047 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
5048 TaskDup->setDoesNotRecurse();
5049 CodeGenFunction CGF(CGM);
5050 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
5051 Loc);
5052
5053 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5054 CGF.GetAddrOfLocalVar(&DstArg),
5055 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5056 // task_dst->liter = lastpriv;
5057 if (WithLastIter) {
5058 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
5059 LValue Base = CGF.EmitLValueForField(
5060 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5061 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
5062 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
5063 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
5064 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
5065 }
5066
5067 // Emit initial values for private copies (if any).
5068 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5068, __PRETTY_FUNCTION__))
;
5069 Address KmpTaskSharedsPtr = Address::invalid();
5070 if (!Data.FirstprivateVars.empty()) {
5071 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5072 CGF.GetAddrOfLocalVar(&SrcArg),
5073 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5074 LValue Base = CGF.EmitLValueForField(
5075 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5076 KmpTaskSharedsPtr = Address(
5077 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
5078 Base, *std::next(KmpTaskTQTyRD->field_begin(),
5079 KmpTaskTShareds)),
5080 Loc),
5081 CGF.getNaturalTypeAlignment(SharedsTy));
5082 }
5083 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
5084 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
5085 CGF.FinishFunction();
5086 return TaskDup;
5087}
5088
5089/// Checks if destructor function is required to be generated.
5090/// \return true if cleanups are required, false otherwise.
5091static bool
5092checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
5093 bool NeedsCleanup = false;
5094 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
5095 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
5096 for (const FieldDecl *FD : PrivateRD->fields()) {
5097 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
5098 if (NeedsCleanup)
5099 break;
5100 }
5101 return NeedsCleanup;
5102}
5103
5104CGOpenMPRuntime::TaskResultTy
5105CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
5106 const OMPExecutableDirective &D,
5107 llvm::Function *TaskFunction, QualType SharedsTy,
5108 Address Shareds, const OMPTaskDataTy &Data) {
5109 ASTContext &C = CGM.getContext();
5110 llvm::SmallVector<PrivateDataTy, 4> Privates;
5111 // Aggregate privates and sort them by the alignment.
5112 auto I = Data.PrivateCopies.begin();
5113 for (const Expr *E : Data.PrivateVars) {
5114 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5115 Privates.emplace_back(
5116 C.getDeclAlign(VD),
5117 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5118 /*PrivateElemInit=*/nullptr));
5119 ++I;
5120 }
5121 I = Data.FirstprivateCopies.begin();
5122 auto IElemInitRef = Data.FirstprivateInits.begin();
5123 for (const Expr *E : Data.FirstprivateVars) {
5124 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5125 Privates.emplace_back(
5126 C.getDeclAlign(VD),
5127 PrivateHelpersTy(
5128 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5129 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
5130 ++I;
5131 ++IElemInitRef;
5132 }
5133 I = Data.LastprivateCopies.begin();
5134 for (const Expr *E : Data.LastprivateVars) {
5135 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5136 Privates.emplace_back(
5137 C.getDeclAlign(VD),
5138 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5139 /*PrivateElemInit=*/nullptr));
5140 ++I;
5141 }
5142 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5143 return L.first > R.first;
5144 });
5145 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5146 // Build type kmp_routine_entry_t (if not built yet).
5147 emitKmpRoutineEntryT(KmpInt32Ty);
5148 // Build type kmp_task_t (if not built yet).
5149 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5150 if (SavedKmpTaskloopTQTy.isNull()) {
5151 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5152 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5153 }
5154 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5155 } else {
5156 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5159, __PRETTY_FUNCTION__))
5157 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5159, __PRETTY_FUNCTION__))
5158 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5159, __PRETTY_FUNCTION__))
5159 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5159, __PRETTY_FUNCTION__))
;
5160 if (SavedKmpTaskTQTy.isNull()) {
5161 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5162 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5163 }
5164 KmpTaskTQTy = SavedKmpTaskTQTy;
5165 }
5166 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5167 // Build particular struct kmp_task_t for the given task.
5168 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5169 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5170 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5171 QualType KmpTaskTWithPrivatesPtrQTy =
5172 C.getPointerType(KmpTaskTWithPrivatesQTy);
5173 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5174 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5175 KmpTaskTWithPrivatesTy->getPointerTo();
5176 llvm::Value *KmpTaskTWithPrivatesTySize =
5177 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5178 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5179
5180 // Emit initial values for private copies (if any).
5181 llvm::Value *TaskPrivatesMap = nullptr;
5182 llvm::Type *TaskPrivatesMapTy =
5183 std::next(TaskFunction->arg_begin(), 3)->getType();
5184 if (!Privates.empty()) {
5185 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5186 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5187 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5188 FI->getType(), Privates);
5189 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5190 TaskPrivatesMap, TaskPrivatesMapTy);
5191 } else {
5192 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5193 cast<llvm::PointerType>(TaskPrivatesMapTy));
5194 }
5195 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5196 // kmp_task_t *tt);
5197 llvm::Function *TaskEntry = emitProxyTaskFunction(
5198 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5199 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5200 TaskPrivatesMap);
5201
5202 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5203 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5204 // kmp_routine_entry_t *task_entry);
5205 // Task flags. Format is taken from
5206 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5207 // description of kmp_tasking_flags struct.
5208 enum {
5209 TiedFlag = 0x1,
5210 FinalFlag = 0x2,
5211 DestructorsFlag = 0x8,
5212 PriorityFlag = 0x20
5213 };
5214 unsigned Flags = Data.Tied ? TiedFlag : 0;
5215 bool NeedsCleanup = false;
5216 if (!Privates.empty()) {
5217 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5218 if (NeedsCleanup)
5219 Flags = Flags | DestructorsFlag;
5220 }
5221 if (Data.Priority.getInt())
5222 Flags = Flags | PriorityFlag;
5223 llvm::Value *TaskFlags =
5224 Data.Final.getPointer()
5225 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5226 CGF.Builder.getInt32(FinalFlag),
5227 CGF.Builder.getInt32(/*C=*/0))
5228 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5229 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5230 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5231 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5232 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5233 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5234 TaskEntry, KmpRoutineEntryPtrTy)};
5235 llvm::Value *NewTask;
5236 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5237 // Check if we have any device clause associated with the directive.
5238 const Expr *Device = nullptr;
5239 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5240 Device = C->getDevice();
5241 // Emit device ID if any otherwise use default value.
5242 llvm::Value *DeviceID;
5243 if (Device)
5244 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5245 CGF.Int64Ty, /*isSigned=*/true);
5246 else
5247 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5248 AllocArgs.push_back(DeviceID);
5249 NewTask = CGF.EmitRuntimeCall(
5250 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5251 } else {
5252 NewTask = CGF.EmitRuntimeCall(
5253 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5254 }
5255 llvm::Value *NewTaskNewTaskTTy =
5256 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5257 NewTask, KmpTaskTWithPrivatesPtrTy);
5258 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5259 KmpTaskTWithPrivatesQTy);
5260 LValue TDBase =
5261 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5262 // Fill the data in the resulting kmp_task_t record.
5263 // Copy shareds if there are any.
5264 Address KmpTaskSharedsPtr = Address::invalid();
5265 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5266 KmpTaskSharedsPtr =
5267 Address(CGF.EmitLoadOfScalar(
5268 CGF.EmitLValueForField(
5269 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5270 KmpTaskTShareds)),
5271 Loc),
5272 CGF.getNaturalTypeAlignment(SharedsTy));
5273 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5274 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5275 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5276 }
5277 // Emit initial values for private copies (if any).
5278 TaskResultTy Result;
5279 if (!Privates.empty()) {
5280 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5281 SharedsTy, SharedsPtrTy, Data, Privates,
5282 /*ForDup=*/false);
5283 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5284 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5285 Result.TaskDupFn = emitTaskDupFunction(
5286 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5287 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5288 /*WithLastIter=*/!Data.LastprivateVars.empty());
5289 }
5290 }
5291 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5292 enum { Priority = 0, Destructors = 1 };
5293 // Provide pointer to function with destructors for privates.
5294 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5295 const RecordDecl *KmpCmplrdataUD =
5296 (*FI)->getType()->getAsUnionType()->getDecl();
5297 if (NeedsCleanup) {
5298 llvm::Value *DestructorFn = emitDestructorsFunction(
5299 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5300 KmpTaskTWithPrivatesQTy);
5301 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5302 LValue DestructorsLV = CGF.EmitLValueForField(
5303 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5304 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5305 DestructorFn, KmpRoutineEntryPtrTy),
5306 DestructorsLV);
5307 }
5308 // Set priority.
5309 if (Data.Priority.getInt()) {
5310 LValue Data2LV = CGF.EmitLValueForField(
5311 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5312 LValue PriorityLV = CGF.EmitLValueForField(
5313 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5314 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5315 }
5316 Result.NewTask = NewTask;
5317 Result.TaskEntry = TaskEntry;
5318 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5319 Result.TDBase = TDBase;
5320 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5321 return Result;
5322}
5323
5324void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5325 const OMPExecutableDirective &D,
5326 llvm::Function *TaskFunction,
5327 QualType SharedsTy, Address Shareds,
5328 const Expr *IfCond,
5329 const OMPTaskDataTy &Data) {
5330 if (!CGF.HaveInsertPoint())
5331 return;
5332
5333 TaskResultTy Result =
5334 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5335 llvm::Value *NewTask = Result.NewTask;
5336 llvm::Function *TaskEntry = Result.TaskEntry;
5337 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5338 LValue TDBase = Result.TDBase;
5339 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5340 ASTContext &C = CGM.getContext();
5341 // Process list of dependences.
5342 Address DependenciesArray = Address::invalid();
5343 unsigned NumDependencies = Data.Dependences.size();
5344 if (NumDependencies) {
5345 // Dependence kind for RTL.
5346 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5347 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5348 RecordDecl *KmpDependInfoRD;
5349 QualType FlagsTy =
5350 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5351 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5352 if (KmpDependInfoTy.isNull()) {
5353 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5354 KmpDependInfoRD->startDefinition();
5355 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5356 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5357 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5358 KmpDependInfoRD->completeDefinition();
5359 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5360 } else {
5361 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5362 }
5363 // Define type kmp_depend_info[<Dependences.size()>];
5364 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5365 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5366 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
5367 // kmp_depend_info[<Dependences.size()>] deps;
5368 DependenciesArray =
5369 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5370 for (unsigned I = 0; I < NumDependencies; ++I) {
5371 const Expr *E = Data.Dependences[I].second;
5372 LValue Addr = CGF.EmitLValue(E);
5373 llvm::Value *Size;
5374 QualType Ty = E->getType();
5375 if (const auto *ASE =
5376 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5377 LValue UpAddrLVal =
5378 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5379 llvm::Value *UpAddr =
5380 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5381 llvm::Value *LowIntPtr =
5382 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5383 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5384 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5385 } else {
5386 Size = CGF.getTypeSize(Ty);
5387 }
5388 LValue Base = CGF.MakeAddrLValue(
5389 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5390 KmpDependInfoTy);
5391 // deps[i].base_addr = &<Dependences[i].second>;
5392 LValue BaseAddrLVal = CGF.EmitLValueForField(
5393 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5394 CGF.EmitStoreOfScalar(
5395 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5396 BaseAddrLVal);
5397 // deps[i].len = sizeof(<Dependences[i].second>);
5398 LValue LenLVal = CGF.EmitLValueForField(
5399 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5400 CGF.EmitStoreOfScalar(Size, LenLVal);
5401 // deps[i].flags = <Dependences[i].first>;
5402 RTLDependenceKindTy DepKind;
5403 switch (Data.Dependences[I].first) {
5404 case OMPC_DEPEND_in:
5405 DepKind = DepIn;
5406 break;
5407 // Out and InOut dependencies must use the same code.
5408 case OMPC_DEPEND_out:
5409 case OMPC_DEPEND_inout:
5410 DepKind = DepInOut;
5411 break;
5412 case OMPC_DEPEND_mutexinoutset:
5413 DepKind = DepMutexInOutSet;
5414 break;
5415 case OMPC_DEPEND_source:
5416 case OMPC_DEPEND_sink:
5417 case OMPC_DEPEND_unknown:
5418 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-10~svn374877/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5418)
;
5419 }
5420 LValue FlagsLVal = CGF.EmitLValueForField(
5421 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5422 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5423 FlagsLVal);
5424 }
5425 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5426 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5427 }
5428
5429 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5430 // libcall.
5431 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5432 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5433 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5434 // list is not empty
5435 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5436 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5437 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5438 llvm::Value *DepTaskArgs[7];
5439 if (NumDependencies) {
5440 DepTaskArgs[0] = UpLoc;
5441 DepTaskArgs[1] = ThreadID;
5442 DepTaskArgs[2] = NewTask;
5443 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5444 DepTaskArgs[4] = DependenciesArray.getPointer();
5445 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5446 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5447 }
5448 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5449 &TaskArgs,
5450 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5451 if (!Data.Tied) {
5452 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5453 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5454 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5455 }
5456 if (NumDependencies) {
5457 CGF.EmitRuntimeCall(
5458 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5459 } else {
5460 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5461 TaskArgs);
5462 }
5463 // Check if parent region is untied and build return for untied task;
5464 if (auto *Region =
5465 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5466 Region->emitUntiedSwitch(CGF);
5467 };
5468
5469 llvm::Value *DepWaitTaskArgs[6];
5470 if (NumDependencies) {
5471 DepWaitTaskArgs[0] = UpLoc;
5472 DepWaitTaskArgs[1] = ThreadID;
5473 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5474 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5475 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5476 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5477 }
5478 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5479 NumDependencies, &DepWaitTaskArgs,
5480 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5481 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5482 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5483 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5484 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5485 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5486 // is specified.
5487 if (NumDependencies)
5488 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5489 DepWaitTaskArgs);
5490 // Call proxy_task_entry(gtid, new_task);
5491 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5492 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5493 Action.Enter(CGF);
5494 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5495 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5496 OutlinedFnArgs);
5497 };
5498
5499 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5500 // kmp_task_t *new_task);
5501 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5502 // kmp_task_t *new_task);
5503 RegionCodeGenTy RCG(CodeGen);
5504 CommonActionTy Action(
5505 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5506 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5507 RCG.setAction(Action);
5508 RCG(CGF);
5509 };
5510
5511 if (IfCond) {
5512 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5513 } else {
5514 RegionCodeGenTy ThenRCG(ThenCodeGen);
5515 ThenRCG(CGF);
5516 }
5517}
5518
5519void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5520 const OMPLoopDirective &D,
5521 llvm::Function *TaskFunction,
5522 QualType SharedsTy, Address Shareds,
5523 const Expr *IfCond,
5524 const OMPTaskDataTy &Data) {
5525 if (!CGF.HaveInsertPoint())
5526 return;
5527 TaskResultTy Result =
5528 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5529 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5530 // libcall.
5531 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5532 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5533 // sched, kmp_uint64 grainsize, void *task_dup);
5534 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5535 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5536 llvm::Value *IfVal;
5537 if (IfCond) {
5538 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5539 /*isSigned=*/true);
5540 } else {
5541 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5542 }
5543
5544 LValue LBLVal = CGF.EmitLValueForField(
5545 Result.TDBase,
5546 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5547 const auto *LBVar =
5548 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5549 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5550 /*IsInitializer=*/true);
5551 LValue UBLVal = CGF.EmitLValueForField(
5552 Result.TDBase,
5553 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5554 const auto *UBVar =
5555 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5556 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5557 /*IsInitializer=*/true);
5558 LValue StLVal = CGF.EmitLValueForField(
5559 Result.TDBase,
5560 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5561 const auto *StVar =
5562 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5563 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5564 /*IsInitializer=*/true);
5565 // Store reductions address.
5566 LValue RedLVal = CGF.EmitLValueForField(
5567 Result.TDBase,
5568 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5569 if (Data.Reductions) {
5570 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5571 } else {
5572 CGF.EmitNullInitialization(RedLVal.getAddress(),
5573 CGF.getContext().VoidPtrTy);
5574 }
5575 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5576 llvm::Value *TaskArgs[] = {
5577 UpLoc,
5578 ThreadID,
5579 Result.NewTask,
5580 IfVal,
5581 LBLVal.getPointer(),
5582 UBLVal.getPointer(),
5583 CGF.EmitLoadOfScalar(StLVal, Loc),
5584 llvm::ConstantInt::getSigned(
5585 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5586 llvm::ConstantInt::getSigned(
5587 CGF.IntTy, Data.Schedule.getPointer()
5588 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5589 : NoSchedule),
5590 Data.Schedule.getPointer()
5591 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5592 /*isSigned=*/false)
5593 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5594 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5595 Result.TaskDupFn, CGF.VoidPtrTy)
5596 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5597 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5598}
5599
5600/// Emit reduction operation for each element of array (required for
5601/// array sections) LHS op = RHS.
5602/// \param Type Type of array.
5603/// \param LHSVar Variable on the left side of the reduction operation
5604/// (references element of array in original variable).
5605/// \param RHSVar Variable on the right side of the reduction operation
5606/// (references element of array in original variable).
5607/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5608/// RHSVar.
5609static void EmitOMPAggregateReduction(
5610 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5611 const VarDecl *RHSVar,
5612 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5613 const Expr *, const Expr *)> &RedOpGen,
5614 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5615 const Expr *UpExpr = nullptr) {
5616 // Perform element-by-element initialization.
5617 QualType ElementTy;
5618 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5619 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5620
5621 // Drill down to the base element type on both arrays.
5622 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5623 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5624
5625 llvm::Value *RHSBegin = RHSAddr.getPointer();
5626 llvm::Value *LHSBegin = LHSAddr.getPointer();
5627 // Cast from pointer to array type to pointer to single element.
5628 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5629 // The basic structure here is a while-do loop.
5630 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5631 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5632 llvm::Value *IsEmpty =
5633 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5634 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5635
5636 // Enter the loop body, making that address the current address.
5637 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5638 CGF.EmitBlock(BodyBB);
5639
5640 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5641
5642 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5643 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5644 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5645 Address RHSElementCurrent =
5646 Address(RHSElementPHI,
5647 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5648
5649 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5650 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5651 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5652 Address LHSElementCurrent =
5653 Address(LHSElementPHI,
5654 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5655
5656 // Emit copy.
5657 CodeGenFunction::OMPPrivateScope Scope(CGF);
5658 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5659 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5660 Scope.Privatize();
5661 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5662 Scope.ForceCleanup();
5663
5664 // Shift the address forward by one element.
5665 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5666 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5667 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5668 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5669 // Check whether we've reached the end.
5670 llvm::Value *Done =
5671 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5672 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5673 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5674 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5675
5676 // Done.
5677 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5678}
5679
5680/// Emit reduction combiner. If the combiner is a simple expression emit it as
5681/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5682/// UDR combiner function.
5683static void emitReductionCombiner(CodeGenFunction &CGF,
5684 const Expr *ReductionOp) {
5685 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5686 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5687 if (const auto *DRE =
5688 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5689 if (const auto *DRD =
5690 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5691 std::pair<llvm::Function *, llvm::Function *> Reduction =
5692 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5693 RValue Func = RValue::get(Reduction.first);
5694 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5695 CGF.EmitIgnoredExpr(ReductionOp);
5696 return;
5697 }
5698 CGF.EmitIgnoredExpr(ReductionOp);
5699}
5700
5701llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5702 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5703 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5704 ArrayRef<const Expr *> ReductionOps) {
5705 ASTContext &C = CGM.getContext();
5706
5707 // void reduction_func(void *LHSArg, void *RHSArg);
5708 FunctionArgList Args;
5709 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5710 ImplicitParamDecl::Other);
5711 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5712 ImplicitParamDecl::Other);
5713 Args.push_back(&LHSArg);
5714 Args.push_back(&RHSArg);
5715 const auto &CGFI =
5716 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5717 std::string Name = getName({"omp", "reduction", "reduction_func"});
5718 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5719 llvm::GlobalValue::InternalLinkage, Name,
5720 &CGM.getModule());
5721 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5722 Fn->setDoesNotRecurse();
5723 CodeGenFunction CGF(CGM);
5724 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5725
5726 // Dst = (void*[n])(LHSArg);
5727 // Src = (void*[n])(RHSArg);
5728 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5729 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5730 ArgsType), CGF.getPointerAlign());
5731 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5732 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5733 ArgsType), CGF.getPointerAlign());
5734
5735 // ...
5736 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5737