Bug Summary

File:tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 2761, column 20
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-9~svn362543/tools/clang/include -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn362543/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn362543=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-06-05-060531-1271-1 -x c++ /build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp -faddrsig

/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGOpenMPRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/CodeGen/ConstantInitBuilder.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/StmtOpenMP.h"
21#include "clang/Basic/BitmaskEnum.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30
31using namespace clang;
32using namespace CodeGen;
33
34namespace {
35/// Base class for handling code generation inside OpenMP regions.
36class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
37public:
38 /// Kinds of OpenMP regions used in codegen.
39 enum CGOpenMPRegionKind {
40 /// Region with outlined function for standalone 'parallel'
41 /// directive.
42 ParallelOutlinedRegion,
43 /// Region with outlined function for standalone 'task' directive.
44 TaskOutlinedRegion,
45 /// Region for constructs that do not require function outlining,
46 /// like 'for', 'sections', 'atomic' etc. directives.
47 InlinedRegion,
48 /// Region with outlined function for standalone 'target' directive.
49 TargetRegion,
50 };
51
52 CGOpenMPRegionInfo(const CapturedStmt &CS,
53 const CGOpenMPRegionKind RegionKind,
54 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
55 bool HasCancel)
56 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58
59 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
60 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61 bool HasCancel)
62 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
64
65 /// Get a variable or parameter for storing global thread id
66 /// inside OpenMP construct.
67 virtual const VarDecl *getThreadIDVariable() const = 0;
68
69 /// Emit the captured statement body.
70 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
71
72 /// Get an LValue for the current ThreadID variable.
73 /// \return LValue for thread id variable. This LValue always has type int32*.
74 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75
76 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
77
78 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
79
80 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
81
82 bool hasCancel() const { return HasCancel; }
83
84 static bool classof(const CGCapturedStmtInfo *Info) {
85 return Info->getKind() == CR_OpenMP;
86 }
87
88 ~CGOpenMPRegionInfo() override = default;
89
90protected:
91 CGOpenMPRegionKind RegionKind;
92 RegionCodeGenTy CodeGen;
93 OpenMPDirectiveKind Kind;
94 bool HasCancel;
95};
96
97/// API for captured statement code generation in OpenMP constructs.
98class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
99public:
100 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
101 const RegionCodeGenTy &CodeGen,
102 OpenMPDirectiveKind Kind, bool HasCancel,
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
105 HasCancel),
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 107, __PRETTY_FUNCTION__))
;
108 }
109
110 /// Get a variable or parameter for storing global thread id
111 /// inside OpenMP construct.
112 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
113
114 /// Get the name of the capture helper.
115 StringRef getHelperName() const override { return HelperName; }
116
117 static bool classof(const CGCapturedStmtInfo *Info) {
118 return CGOpenMPRegionInfo::classof(Info) &&
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
121 }
122
123private:
124 /// A variable or parameter storing global thread id for OpenMP
125 /// constructs.
126 const VarDecl *ThreadIDVar;
127 StringRef HelperName;
128};
129
130/// API for captured statement code generation in OpenMP constructs.
131class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
132public:
133 class UntiedTaskActionTy final : public PrePostActionTy {
134 bool Untied;
135 const VarDecl *PartIDVar;
136 const RegionCodeGenTy UntiedCodeGen;
137 llvm::SwitchInst *UntiedSwitch = nullptr;
138
139 public:
140 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
141 const RegionCodeGenTy &UntiedCodeGen)
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
143 void Enter(CodeGenFunction &CGF) override {
144 if (Untied) {
145 // Emit task switching point.
146 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
147 CGF.GetAddrOfLocalVar(PartIDVar),
148 PartIDVar->getType()->castAs<PointerType>());
149 llvm::Value *Res =
150 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
151 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
152 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153 CGF.EmitBlock(DoneBB);
154 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157 CGF.Builder.GetInsertBlock());
158 emitUntiedSwitch(CGF);
159 }
160 }
161 void emitUntiedSwitch(CodeGenFunction &CGF) const {
162 if (Untied) {
163 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
164 CGF.GetAddrOfLocalVar(PartIDVar),
165 PartIDVar->getType()->castAs<PointerType>());
166 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167 PartIdLVal);
168 UntiedCodeGen(CGF);
169 CodeGenFunction::JumpDest CurPoint =
170 CGF.getJumpDestInCurrentScope(".untied.next.");
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174 CGF.Builder.GetInsertBlock());
175 CGF.EmitBranchThroughCleanup(CurPoint);
176 CGF.EmitBlock(CurPoint.getBlock());
177 }
178 }
179 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180 };
181 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
182 const VarDecl *ThreadIDVar,
183 const RegionCodeGenTy &CodeGen,
184 OpenMPDirectiveKind Kind, bool HasCancel,
185 const UntiedTaskActionTy &Action)
186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187 ThreadIDVar(ThreadIDVar), Action(Action) {
188 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 188, __PRETTY_FUNCTION__))
;
189 }
190
191 /// Get a variable or parameter for storing global thread id
192 /// inside OpenMP construct.
193 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
194
195 /// Get an LValue for the current ThreadID variable.
196 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197
198 /// Get the name of the capture helper.
199 StringRef getHelperName() const override { return ".omp_outlined."; }
200
201 void emitUntiedSwitch(CodeGenFunction &CGF) override {
202 Action.emitUntiedSwitch(CGF);
203 }
204
205 static bool classof(const CGCapturedStmtInfo *Info) {
206 return CGOpenMPRegionInfo::classof(Info) &&
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208 TaskOutlinedRegion;
209 }
210
211private:
212 /// A variable or parameter storing global thread id for OpenMP
213 /// constructs.
214 const VarDecl *ThreadIDVar;
215 /// Action for emitting code for untied tasks.
216 const UntiedTaskActionTy &Action;
217};
218
219/// API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
223 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
224 const RegionCodeGenTy &CodeGen,
225 OpenMPDirectiveKind Kind, bool HasCancel)
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OldCSI(OldCSI),
228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229
230 // Retrieve the value of the context parameter.
231 llvm::Value *getContextValue() const override {
232 if (OuterRegionInfo)
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 234)
;
235 }
236
237 void setContextValue(llvm::Value *V) override {
238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
240 return;
241 }
242 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 242)
;
243 }
244
245 /// Lookup the captured field decl for a variable.
246 const FieldDecl *lookup(const VarDecl *VD) const override {
247 if (OuterRegionInfo)
248 return OuterRegionInfo->lookup(VD);
249 // If there is no outer outlined region,no need to lookup in a list of
250 // captured variables, we can use the original one.
251 return nullptr;
252 }
253
254 FieldDecl *getThisFieldDecl() const override {
255 if (OuterRegionInfo)
256 return OuterRegionInfo->getThisFieldDecl();
257 return nullptr;
258 }
259
260 /// Get a variable or parameter for storing global thread id
261 /// inside OpenMP construct.
262 const VarDecl *getThreadIDVariable() const override {
263 if (OuterRegionInfo)
264 return OuterRegionInfo->getThreadIDVariable();
265 return nullptr;
266 }
267
268 /// Get an LValue for the current ThreadID variable.
269 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270 if (OuterRegionInfo)
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 272)
;
273 }
274
275 /// Get the name of the capture helper.
276 StringRef getHelperName() const override {
277 if (auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
279 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 279)
;
280 }
281
282 void emitUntiedSwitch(CodeGenFunction &CGF) override {
283 if (OuterRegionInfo)
284 OuterRegionInfo->emitUntiedSwitch(CGF);
285 }
286
287 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
289 static bool classof(const CGCapturedStmtInfo *Info) {
290 return CGOpenMPRegionInfo::classof(Info) &&
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292 }
293
294 ~CGOpenMPInlinedRegionInfo() override = default;
295
296private:
297 /// CodeGen info about outer OpenMP region.
298 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299 CGOpenMPRegionInfo *OuterRegionInfo;
300};
301
302/// API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
308public:
309 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
310 const RegionCodeGenTy &CodeGen, StringRef HelperName)
311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
312 /*HasCancel=*/false),
313 HelperName(HelperName) {}
314
315 /// This is unused for target regions because each starts executing
316 /// with a single thread.
317 const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319 /// Get the name of the capture helper.
320 StringRef getHelperName() const override { return HelperName; }
321
322 static bool classof(const CGCapturedStmtInfo *Info) {
323 return CGOpenMPRegionInfo::classof(Info) &&
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325 }
326
327private:
328 StringRef HelperName;
329};
330
331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
332 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 332)
;
333}
334/// API for generation of expressions captured in a innermost OpenMP
335/// region.
336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
337public:
338 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340 OMPD_unknown,
341 /*HasCancel=*/false),
342 PrivScope(CGF) {
343 // Make sure the globals captured in the provided statement are local by
344 // using the privatization logic. We assume the same variable is not
345 // captured more than once.
346 for (const auto &C : CS.captures()) {
347 if (!C.capturesVariable() && !C.capturesVariableByCopy())
348 continue;
349
350 const VarDecl *VD = C.getCapturedVar();
351 if (VD->isLocalVarDeclOrParm())
352 continue;
353
354 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
355 /*RefersToEnclosingVariableOrCapture=*/false,
356 VD->getType().getNonReferenceType(), VK_LValue,
357 C.getLocation());
358 PrivScope.addPrivate(
359 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
360 }
361 (void)PrivScope.Privatize();
362 }
363
364 /// Lookup the captured field decl for a variable.
365 const FieldDecl *lookup(const VarDecl *VD) const override {
366 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
367 return FD;
368 return nullptr;
369 }
370
371 /// Emit the captured statement body.
372 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
373 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 373)
;
374 }
375
376 /// Get a variable or parameter for storing global thread id
377 /// inside OpenMP construct.
378 const VarDecl *getThreadIDVariable() const override {
379 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 379)
;
380 }
381
382 /// Get the name of the capture helper.
383 StringRef getHelperName() const override {
384 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 384)
;
385 }
386
387 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
388
389private:
390 /// Private scope to capture global variables.
391 CodeGenFunction::OMPPrivateScope PrivScope;
392};
393
394/// RAII for emitting code of OpenMP constructs.
395class InlinedOpenMPRegionRAII {
396 CodeGenFunction &CGF;
397 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398 FieldDecl *LambdaThisCaptureField = nullptr;
399 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
400
401public:
402 /// Constructs region for combined constructs.
403 /// \param CodeGen Code generation sequence for combined directives. Includes
404 /// a list of functions used for code generation of implicitly inlined
405 /// regions.
406 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
407 OpenMPDirectiveKind Kind, bool HasCancel)
408 : CGF(CGF) {
409 // Start emission for the construct.
410 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414 CGF.LambdaThisCaptureField = nullptr;
415 BlockInfo = CGF.BlockInfo;
416 CGF.BlockInfo = nullptr;
417 }
418
419 ~InlinedOpenMPRegionRAII() {
420 // Restore original CapturedStmtInfo only if we're done with code emission.
421 auto *OldCSI =
422 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
423 delete CGF.CapturedStmtInfo;
424 CGF.CapturedStmtInfo = OldCSI;
425 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
426 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
427 CGF.BlockInfo = BlockInfo;
428 }
429};
430
431/// Values for bit flags used in the ident_t to describe the fields.
432/// All enumeric elements are named and described in accordance with the code
433/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
434enum OpenMPLocationFlags : unsigned {
435 /// Use trampoline for internal microtask.
436 OMP_IDENT_IMD = 0x01,
437 /// Use c-style ident structure.
438 OMP_IDENT_KMPC = 0x02,
439 /// Atomic reduction option for kmpc_reduce.
440 OMP_ATOMIC_REDUCE = 0x10,
441 /// Explicit 'barrier' directive.
442 OMP_IDENT_BARRIER_EXPL = 0x20,
443 /// Implicit barrier in code.
444 OMP_IDENT_BARRIER_IMPL = 0x40,
445 /// Implicit barrier in 'for' directive.
446 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
447 /// Implicit barrier in 'sections' directive.
448 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
449 /// Implicit barrier in 'single' directive.
450 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
451 /// Call of __kmp_for_static_init for static loop.
452 OMP_IDENT_WORK_LOOP = 0x200,
453 /// Call of __kmp_for_static_init for sections.
454 OMP_IDENT_WORK_SECTIONS = 0x400,
455 /// Call of __kmp_for_static_init for distribute.
456 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
458};
459
460namespace {
461LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
462/// Values for bit flags for marking which requires clauses have been used.
463enum OpenMPOffloadingRequiresDirFlags : int64_t {
464 /// flag undefined.
465 OMP_REQ_UNDEFINED = 0x000,
466 /// no requires clause present.
467 OMP_REQ_NONE = 0x001,
468 /// reverse_offload clause.
469 OMP_REQ_REVERSE_OFFLOAD = 0x002,
470 /// unified_address clause.
471 OMP_REQ_UNIFIED_ADDRESS = 0x004,
472 /// unified_shared_memory clause.
473 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
474 /// dynamic_allocators clause.
475 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
476 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
477};
478} // anonymous namespace
479
480/// Describes ident structure that describes a source location.
481/// All descriptions are taken from
482/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
483/// Original structure:
484/// typedef struct ident {
485/// kmp_int32 reserved_1; /**< might be used in Fortran;
486/// see above */
487/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
488/// KMP_IDENT_KMPC identifies this union
489/// member */
490/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
491/// see above */
492///#if USE_ITT_BUILD
493/// /* but currently used for storing
494/// region-specific ITT */
495/// /* contextual information. */
496///#endif /* USE_ITT_BUILD */
497/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
498/// C++ */
499/// char const *psource; /**< String describing the source location.
500/// The string is composed of semi-colon separated
501// fields which describe the source file,
502/// the function and a pair of line numbers that
503/// delimit the construct.
504/// */
505/// } ident_t;
506enum IdentFieldIndex {
507 /// might be used in Fortran
508 IdentField_Reserved_1,
509 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
510 IdentField_Flags,
511 /// Not really used in Fortran any more
512 IdentField_Reserved_2,
513 /// Source[4] in Fortran, do not use for C++
514 IdentField_Reserved_3,
515 /// String describing the source location. The string is composed of
516 /// semi-colon separated fields which describe the source file, the function
517 /// and a pair of line numbers that delimit the construct.
518 IdentField_PSource
519};
520
521/// Schedule types for 'omp for' loops (these enumerators are taken from
522/// the enum sched_type in kmp.h).
523enum OpenMPSchedType {
524 /// Lower bound for default (unordered) versions.
525 OMP_sch_lower = 32,
526 OMP_sch_static_chunked = 33,
527 OMP_sch_static = 34,
528 OMP_sch_dynamic_chunked = 35,
529 OMP_sch_guided_chunked = 36,
530 OMP_sch_runtime = 37,
531 OMP_sch_auto = 38,
532 /// static with chunk adjustment (e.g., simd)
533 OMP_sch_static_balanced_chunked = 45,
534 /// Lower bound for 'ordered' versions.
535 OMP_ord_lower = 64,
536 OMP_ord_static_chunked = 65,
537 OMP_ord_static = 66,
538 OMP_ord_dynamic_chunked = 67,
539 OMP_ord_guided_chunked = 68,
540 OMP_ord_runtime = 69,
541 OMP_ord_auto = 70,
542 OMP_sch_default = OMP_sch_static,
543 /// dist_schedule types
544 OMP_dist_sch_static_chunked = 91,
545 OMP_dist_sch_static = 92,
546 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
547 /// Set if the monotonic schedule modifier was present.
548 OMP_sch_modifier_monotonic = (1 << 29),
549 /// Set if the nonmonotonic schedule modifier was present.
550 OMP_sch_modifier_nonmonotonic = (1 << 30),
551};
552
553enum OpenMPRTLFunction {
554 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
555 /// kmpc_micro microtask, ...);
556 OMPRTL__kmpc_fork_call,
557 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
558 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
559 OMPRTL__kmpc_threadprivate_cached,
560 /// Call to void __kmpc_threadprivate_register( ident_t *,
561 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
562 OMPRTL__kmpc_threadprivate_register,
563 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
564 OMPRTL__kmpc_global_thread_num,
565 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
566 // kmp_critical_name *crit);
567 OMPRTL__kmpc_critical,
568 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
569 // global_tid, kmp_critical_name *crit, uintptr_t hint);
570 OMPRTL__kmpc_critical_with_hint,
571 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
572 // kmp_critical_name *crit);
573 OMPRTL__kmpc_end_critical,
574 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
575 // global_tid);
576 OMPRTL__kmpc_cancel_barrier,
577 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
578 OMPRTL__kmpc_barrier,
579 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
580 OMPRTL__kmpc_for_static_fini,
581 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
582 // global_tid);
583 OMPRTL__kmpc_serialized_parallel,
584 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
585 // global_tid);
586 OMPRTL__kmpc_end_serialized_parallel,
587 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
588 // kmp_int32 num_threads);
589 OMPRTL__kmpc_push_num_threads,
590 // Call to void __kmpc_flush(ident_t *loc);
591 OMPRTL__kmpc_flush,
592 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
593 OMPRTL__kmpc_master,
594 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
595 OMPRTL__kmpc_end_master,
596 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
597 // int end_part);
598 OMPRTL__kmpc_omp_taskyield,
599 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
600 OMPRTL__kmpc_single,
601 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
602 OMPRTL__kmpc_end_single,
603 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
604 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
605 // kmp_routine_entry_t *task_entry);
606 OMPRTL__kmpc_omp_task_alloc,
607 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
608 // new_task);
609 OMPRTL__kmpc_omp_task,
610 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
611 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
612 // kmp_int32 didit);
613 OMPRTL__kmpc_copyprivate,
614 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
615 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
616 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
617 OMPRTL__kmpc_reduce,
618 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
619 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
620 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
621 // *lck);
622 OMPRTL__kmpc_reduce_nowait,
623 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
624 // kmp_critical_name *lck);
625 OMPRTL__kmpc_end_reduce,
626 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
627 // kmp_critical_name *lck);
628 OMPRTL__kmpc_end_reduce_nowait,
629 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
630 // kmp_task_t * new_task);
631 OMPRTL__kmpc_omp_task_begin_if0,
632 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
633 // kmp_task_t * new_task);
634 OMPRTL__kmpc_omp_task_complete_if0,
635 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
636 OMPRTL__kmpc_ordered,
637 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
638 OMPRTL__kmpc_end_ordered,
639 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
640 // global_tid);
641 OMPRTL__kmpc_omp_taskwait,
642 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
643 OMPRTL__kmpc_taskgroup,
644 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
645 OMPRTL__kmpc_end_taskgroup,
646 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
647 // int proc_bind);
648 OMPRTL__kmpc_push_proc_bind,
649 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
650 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
651 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
652 OMPRTL__kmpc_omp_task_with_deps,
653 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
654 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
655 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
656 OMPRTL__kmpc_omp_wait_deps,
657 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
658 // global_tid, kmp_int32 cncl_kind);
659 OMPRTL__kmpc_cancellationpoint,
660 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
661 // kmp_int32 cncl_kind);
662 OMPRTL__kmpc_cancel,
663 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
664 // kmp_int32 num_teams, kmp_int32 thread_limit);
665 OMPRTL__kmpc_push_num_teams,
666 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
667 // microtask, ...);
668 OMPRTL__kmpc_fork_teams,
669 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
670 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
671 // sched, kmp_uint64 grainsize, void *task_dup);
672 OMPRTL__kmpc_taskloop,
673 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
674 // num_dims, struct kmp_dim *dims);
675 OMPRTL__kmpc_doacross_init,
676 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
677 OMPRTL__kmpc_doacross_fini,
678 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
679 // *vec);
680 OMPRTL__kmpc_doacross_post,
681 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
682 // *vec);
683 OMPRTL__kmpc_doacross_wait,
684 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
685 // *data);
686 OMPRTL__kmpc_task_reduction_init,
687 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
688 // *d);
689 OMPRTL__kmpc_task_reduction_get_th_data,
690 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
691 OMPRTL__kmpc_alloc,
692 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
693 OMPRTL__kmpc_free,
694
695 //
696 // Offloading related calls
697 //
698 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
699 // size);
700 OMPRTL__kmpc_push_target_tripcount,
701 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
702 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
703 // *arg_types);
704 OMPRTL__tgt_target,
705 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
706 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
707 // *arg_types);
708 OMPRTL__tgt_target_nowait,
709 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
710 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
711 // *arg_types, int32_t num_teams, int32_t thread_limit);
712 OMPRTL__tgt_target_teams,
713 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
714 // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
715 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
716 OMPRTL__tgt_target_teams_nowait,
717 // Call to void __tgt_register_requires(int64_t flags);
718 OMPRTL__tgt_register_requires,
719 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
720 OMPRTL__tgt_register_lib,
721 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
722 OMPRTL__tgt_unregister_lib,
723 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
724 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
725 OMPRTL__tgt_target_data_begin,
726 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
727 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
728 // *arg_types);
729 OMPRTL__tgt_target_data_begin_nowait,
730 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
731 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
732 OMPRTL__tgt_target_data_end,
733 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
734 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
735 // *arg_types);
736 OMPRTL__tgt_target_data_end_nowait,
737 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
738 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
739 OMPRTL__tgt_target_data_update,
740 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
741 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
742 // *arg_types);
743 OMPRTL__tgt_target_data_update_nowait,
744};
745
746/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
747/// region.
748class CleanupTy final : public EHScopeStack::Cleanup {
749 PrePostActionTy *Action;
750
751public:
752 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
753 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
754 if (!CGF.HaveInsertPoint())
755 return;
756 Action->Exit(CGF);
757 }
758};
759
760} // anonymous namespace
761
762void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
763 CodeGenFunction::RunCleanupsScope Scope(CGF);
764 if (PrePostAction) {
765 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
766 Callback(CodeGen, CGF, *PrePostAction);
767 } else {
768 PrePostActionTy Action;
769 Callback(CodeGen, CGF, Action);
770 }
771}
772
773/// Check if the combiner is a call to UDR combiner and if it is so return the
774/// UDR decl used for reduction.
775static const OMPDeclareReductionDecl *
776getReductionInit(const Expr *ReductionOp) {
777 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
778 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
779 if (const auto *DRE =
780 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
781 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
782 return DRD;
783 return nullptr;
784}
785
786static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
787 const OMPDeclareReductionDecl *DRD,
788 const Expr *InitOp,
789 Address Private, Address Original,
790 QualType Ty) {
791 if (DRD->getInitializer()) {
792 std::pair<llvm::Function *, llvm::Function *> Reduction =
793 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
794 const auto *CE = cast<CallExpr>(InitOp);
795 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
796 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
797 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
798 const auto *LHSDRE =
799 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
800 const auto *RHSDRE =
801 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
802 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
803 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
804 [=]() { return Private; });
805 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
806 [=]() { return Original; });
807 (void)PrivateScope.Privatize();
808 RValue Func = RValue::get(Reduction.second);
809 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
810 CGF.EmitIgnoredExpr(InitOp);
811 } else {
812 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
813 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
814 auto *GV = new llvm::GlobalVariable(
815 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
816 llvm::GlobalValue::PrivateLinkage, Init, Name);
817 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
818 RValue InitRVal;
819 switch (CGF.getEvaluationKind(Ty)) {
820 case TEK_Scalar:
821 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
822 break;
823 case TEK_Complex:
824 InitRVal =
825 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
826 break;
827 case TEK_Aggregate:
828 InitRVal = RValue::getAggregate(LV.getAddress());
829 break;
830 }
831 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
832 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
833 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
834 /*IsInitializer=*/false);
835 }
836}
837
838/// Emit initialization of arrays of complex types.
839/// \param DestAddr Address of the array.
840/// \param Type Type of array.
841/// \param Init Initial expression of array.
842/// \param SrcAddr Address of the original array.
843static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
844 QualType Type, bool EmitDeclareReductionInit,
845 const Expr *Init,
846 const OMPDeclareReductionDecl *DRD,
847 Address SrcAddr = Address::invalid()) {
848 // Perform element-by-element initialization.
849 QualType ElementTy;
850
851 // Drill down to the base element type on both arrays.
852 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
853 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
854 DestAddr =
855 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
856 if (DRD)
857 SrcAddr =
858 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
859
860 llvm::Value *SrcBegin = nullptr;
861 if (DRD)
862 SrcBegin = SrcAddr.getPointer();
863 llvm::Value *DestBegin = DestAddr.getPointer();
864 // Cast from pointer to array type to pointer to single element.
865 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
866 // The basic structure here is a while-do loop.
867 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
868 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
869 llvm::Value *IsEmpty =
870 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
871 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
872
873 // Enter the loop body, making that address the current address.
874 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
875 CGF.EmitBlock(BodyBB);
876
877 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
878
879 llvm::PHINode *SrcElementPHI = nullptr;
880 Address SrcElementCurrent = Address::invalid();
881 if (DRD) {
882 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
883 "omp.arraycpy.srcElementPast");
884 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
885 SrcElementCurrent =
886 Address(SrcElementPHI,
887 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
888 }
889 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
890 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
891 DestElementPHI->addIncoming(DestBegin, EntryBB);
892 Address DestElementCurrent =
893 Address(DestElementPHI,
894 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
895
896 // Emit copy.
897 {
898 CodeGenFunction::RunCleanupsScope InitScope(CGF);
899 if (EmitDeclareReductionInit) {
900 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
901 SrcElementCurrent, ElementTy);
902 } else
903 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
904 /*IsInitializer=*/false);
905 }
906
907 if (DRD) {
908 // Shift the address forward by one element.
909 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
910 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
911 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
912 }
913
914 // Shift the address forward by one element.
915 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
916 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
917 // Check whether we've reached the end.
918 llvm::Value *Done =
919 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
920 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
921 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
922
923 // Done.
924 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
925}
926
927LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
928 return CGF.EmitOMPSharedLValue(E);
929}
930
931LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
932 const Expr *E) {
933 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
934 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
935 return LValue();
936}
937
938void ReductionCodeGen::emitAggregateInitialization(
939 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
940 const OMPDeclareReductionDecl *DRD) {
941 // Emit VarDecl with copy init for arrays.
942 // Get the address of the original variable captured in current
943 // captured region.
944 const auto *PrivateVD =
945 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
946 bool EmitDeclareReductionInit =
947 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
948 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
949 EmitDeclareReductionInit,
950 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
951 : PrivateVD->getInit(),
952 DRD, SharedLVal.getAddress());
953}
954
955ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
956 ArrayRef<const Expr *> Privates,
957 ArrayRef<const Expr *> ReductionOps) {
958 ClausesData.reserve(Shareds.size());
959 SharedAddresses.reserve(Shareds.size());
960 Sizes.reserve(Shareds.size());
961 BaseDecls.reserve(Shareds.size());
962 auto IPriv = Privates.begin();
963 auto IRed = ReductionOps.begin();
964 for (const Expr *Ref : Shareds) {
965 ClausesData.emplace_back(Ref, *IPriv, *IRed);
966 std::advance(IPriv, 1);
967 std::advance(IRed, 1);
968 }
969}
970
971void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
972 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 973, __PRETTY_FUNCTION__))
973 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 973, __PRETTY_FUNCTION__))
;
974 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
975 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
976 SharedAddresses.emplace_back(First, Second);
977}
978
979void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
980 const auto *PrivateVD =
981 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
982 QualType PrivateType = PrivateVD->getType();
983 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
984 if (!PrivateType->isVariablyModifiedType()) {
985 Sizes.emplace_back(
986 CGF.getTypeSize(
987 SharedAddresses[N].first.getType().getNonReferenceType()),
988 nullptr);
989 return;
990 }
991 llvm::Value *Size;
992 llvm::Value *SizeInChars;
993 auto *ElemType =
994 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
995 ->getElementType();
996 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
997 if (AsArraySection) {
998 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
999 SharedAddresses[N].first.getPointer());
1000 Size = CGF.Builder.CreateNUWAdd(
1001 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1002 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1003 } else {
1004 SizeInChars = CGF.getTypeSize(
1005 SharedAddresses[N].first.getType().getNonReferenceType());
1006 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1007 }
1008 Sizes.emplace_back(SizeInChars, Size);
1009 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1010 CGF,
1011 cast<OpaqueValueExpr>(
1012 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1013 RValue::get(Size));
1014 CGF.EmitVariablyModifiedType(PrivateType);
1015}
1016
1017void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1018 llvm::Value *Size) {
1019 const auto *PrivateVD =
1020 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1021 QualType PrivateType = PrivateVD->getType();
1022 if (!PrivateType->isVariablyModifiedType()) {
1023 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1025, __PRETTY_FUNCTION__))
1024 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1025, __PRETTY_FUNCTION__))
1025 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1025, __PRETTY_FUNCTION__))
;
1026 return;
1027 }
1028 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1029 CGF,
1030 cast<OpaqueValueExpr>(
1031 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1032 RValue::get(Size));
1033 CGF.EmitVariablyModifiedType(PrivateType);
1034}
1035
1036void ReductionCodeGen::emitInitialization(
1037 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1038 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1039 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1039, __PRETTY_FUNCTION__))
;
1040 const auto *PrivateVD =
1041 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1042 const OMPDeclareReductionDecl *DRD =
1043 getReductionInit(ClausesData[N].ReductionOp);
1044 QualType PrivateType = PrivateVD->getType();
1045 PrivateAddr = CGF.Builder.CreateElementBitCast(
1046 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1047 QualType SharedType = SharedAddresses[N].first.getType();
1048 SharedLVal = CGF.MakeAddrLValue(
1049 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1050 CGF.ConvertTypeForMem(SharedType)),
1051 SharedType, SharedAddresses[N].first.getBaseInfo(),
1052 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1053 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1054 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1055 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1056 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1057 PrivateAddr, SharedLVal.getAddress(),
1058 SharedLVal.getType());
1059 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1060 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1061 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1062 PrivateVD->getType().getQualifiers(),
1063 /*IsInitializer=*/false);
1064 }
1065}
1066
1067bool ReductionCodeGen::needCleanups(unsigned N) {
1068 const auto *PrivateVD =
1069 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1070 QualType PrivateType = PrivateVD->getType();
1071 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1072 return DTorKind != QualType::DK_none;
1073}
1074
1075void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1076 Address PrivateAddr) {
1077 const auto *PrivateVD =
1078 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1079 QualType PrivateType = PrivateVD->getType();
1080 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1081 if (needCleanups(N)) {
1082 PrivateAddr = CGF.Builder.CreateElementBitCast(
1083 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1084 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1085 }
1086}
1087
1088static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1089 LValue BaseLV) {
1090 BaseTy = BaseTy.getNonReferenceType();
1091 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1092 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1093 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1094 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1095 } else {
1096 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1097 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1098 }
1099 BaseTy = BaseTy->getPointeeType();
1100 }
1101 return CGF.MakeAddrLValue(
1102 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1103 CGF.ConvertTypeForMem(ElTy)),
1104 BaseLV.getType(), BaseLV.getBaseInfo(),
1105 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1106}
1107
1108static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1109 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1110 llvm::Value *Addr) {
1111 Address Tmp = Address::invalid();
1112 Address TopTmp = Address::invalid();
1113 Address MostTopTmp = Address::invalid();
1114 BaseTy = BaseTy.getNonReferenceType();
1115 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1116 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1117 Tmp = CGF.CreateMemTemp(BaseTy);
1118 if (TopTmp.isValid())
1119 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1120 else
1121 MostTopTmp = Tmp;
1122 TopTmp = Tmp;
1123 BaseTy = BaseTy->getPointeeType();
1124 }
1125 llvm::Type *Ty = BaseLVType;
1126 if (Tmp.isValid())
1127 Ty = Tmp.getElementType();
1128 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1129 if (Tmp.isValid()) {
1130 CGF.Builder.CreateStore(Addr, Tmp);
1131 return MostTopTmp;
1132 }
1133 return Address(Addr, BaseLVAlignment);
1134}
1135
1136static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1137 const VarDecl *OrigVD = nullptr;
1138 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1139 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1140 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1141 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1142 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1143 Base = TempASE->getBase()->IgnoreParenImpCasts();
1144 DE = cast<DeclRefExpr>(Base);
1145 OrigVD = cast<VarDecl>(DE->getDecl());
1146 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1147 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1148 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1149 Base = TempASE->getBase()->IgnoreParenImpCasts();
1150 DE = cast<DeclRefExpr>(Base);
1151 OrigVD = cast<VarDecl>(DE->getDecl());
1152 }
1153 return OrigVD;
1154}
1155
1156Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1157 Address PrivateAddr) {
1158 const DeclRefExpr *DE;
1159 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1160 BaseDecls.emplace_back(OrigVD);
1161 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1162 LValue BaseLValue =
1163 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1164 OriginalBaseLValue);
1165 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1166 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1167 llvm::Value *PrivatePointer =
1168 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1169 PrivateAddr.getPointer(),
1170 SharedAddresses[N].first.getAddress().getType());
1171 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1172 return castToBase(CGF, OrigVD->getType(),
1173 SharedAddresses[N].first.getType(),
1174 OriginalBaseLValue.getAddress().getType(),
1175 OriginalBaseLValue.getAlignment(), Ptr);
1176 }
1177 BaseDecls.emplace_back(
1178 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1179 return PrivateAddr;
1180}
1181
1182bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1183 const OMPDeclareReductionDecl *DRD =
1184 getReductionInit(ClausesData[N].ReductionOp);
1185 return DRD && DRD->getInitializer();
1186}
1187
1188LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1189 return CGF.EmitLoadOfPointerLValue(
1190 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1191 getThreadIDVariable()->getType()->castAs<PointerType>());
1192}
1193
1194void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1195 if (!CGF.HaveInsertPoint())
1196 return;
1197 // 1.2.2 OpenMP Language Terminology
1198 // Structured block - An executable statement with a single entry at the
1199 // top and a single exit at the bottom.
1200 // The point of exit cannot be a branch out of the structured block.
1201 // longjmp() and throw() must not violate the entry/exit criteria.
1202 CGF.EHStack.pushTerminate();
1203 CodeGen(CGF);
1204 CGF.EHStack.popTerminate();
1205}
1206
1207LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1208 CodeGenFunction &CGF) {
1209 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1210 getThreadIDVariable()->getType(),
1211 AlignmentSource::Decl);
1212}
1213
1214static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1215 QualType FieldTy) {
1216 auto *Field = FieldDecl::Create(
1217 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1218 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1219 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1220 Field->setAccess(AS_public);
1221 DC->addDecl(Field);
1222 return Field;
1223}
1224
1225CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1226 StringRef Separator)
1227 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1228 OffloadEntriesInfoManager(CGM) {
1229 ASTContext &C = CGM.getContext();
1230 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1231 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1232 RD->startDefinition();
1233 // reserved_1
1234 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1235 // flags
1236 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1237 // reserved_2
1238 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1239 // reserved_3
1240 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1241 // psource
1242 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1243 RD->completeDefinition();
1244 IdentQTy = C.getRecordType(RD);
1245 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1246 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1247
1248 loadOffloadInfoMetadata();
1249}
1250
1251void CGOpenMPRuntime::clear() {
1252 InternalVars.clear();
1253 // Clean non-target variable declarations possibly used only in debug info.
1254 for (const auto &Data : EmittedNonTargetVariables) {
1255 if (!Data.getValue().pointsToAliveValue())
1256 continue;
1257 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1258 if (!GV)
1259 continue;
1260 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1261 continue;
1262 GV->eraseFromParent();
1263 }
1264}
1265
1266std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1267 SmallString<128> Buffer;
1268 llvm::raw_svector_ostream OS(Buffer);
1269 StringRef Sep = FirstSeparator;
1270 for (StringRef Part : Parts) {
1271 OS << Sep << Part;
1272 Sep = Separator;
1273 }
1274 return OS.str();
1275}
1276
1277static llvm::Function *
1278emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1279 const Expr *CombinerInitializer, const VarDecl *In,
1280 const VarDecl *Out, bool IsCombiner) {
1281 // void .omp_combiner.(Ty *in, Ty *out);
1282 ASTContext &C = CGM.getContext();
1283 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1284 FunctionArgList Args;
1285 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1286 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1287 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1288 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1289 Args.push_back(&OmpOutParm);
1290 Args.push_back(&OmpInParm);
1291 const CGFunctionInfo &FnInfo =
1292 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1293 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1294 std::string Name = CGM.getOpenMPRuntime().getName(
1295 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1296 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1297 Name, &CGM.getModule());
1298 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1299 if (CGM.getLangOpts().Optimize) {
1300 Fn->removeFnAttr(llvm::Attribute::NoInline);
1301 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1302 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1303 }
1304 CodeGenFunction CGF(CGM);
1305 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1306 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1307 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1308 Out->getLocation());
1309 CodeGenFunction::OMPPrivateScope Scope(CGF);
1310 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1311 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1312 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1313 .getAddress();
1314 });
1315 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1316 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1317 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1318 .getAddress();
1319 });
1320 (void)Scope.Privatize();
1321 if (!IsCombiner && Out->hasInit() &&
1322 !CGF.isTrivialInitializer(Out->getInit())) {
1323 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1324 Out->getType().getQualifiers(),
1325 /*IsInitializer=*/true);
1326 }
1327 if (CombinerInitializer)
1328 CGF.EmitIgnoredExpr(CombinerInitializer);
1329 Scope.ForceCleanup();
1330 CGF.FinishFunction();
1331 return Fn;
1332}
1333
1334void CGOpenMPRuntime::emitUserDefinedReduction(
1335 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1336 if (UDRMap.count(D) > 0)
1337 return;
1338 llvm::Function *Combiner = emitCombinerOrInitializer(
1339 CGM, D->getType(), D->getCombiner(),
1340 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1341 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1342 /*IsCombiner=*/true);
1343 llvm::Function *Initializer = nullptr;
1344 if (const Expr *Init = D->getInitializer()) {
1345 Initializer = emitCombinerOrInitializer(
1346 CGM, D->getType(),
1347 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1348 : nullptr,
1349 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1350 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1351 /*IsCombiner=*/false);
1352 }
1353 UDRMap.try_emplace(D, Combiner, Initializer);
1354 if (CGF) {
1355 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1356 Decls.second.push_back(D);
1357 }
1358}
1359
1360std::pair<llvm::Function *, llvm::Function *>
1361CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1362 auto I = UDRMap.find(D);
1363 if (I != UDRMap.end())
1364 return I->second;
1365 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1366 return UDRMap.lookup(D);
1367}
1368
1369static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1370 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1371 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1372 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1373 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1374, __PRETTY_FUNCTION__))
1374 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1374, __PRETTY_FUNCTION__))
;
1375 CodeGenFunction CGF(CGM, true);
1376 bool HasCancel = false;
1377 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1378 HasCancel = OPD->hasCancel();
1379 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1380 HasCancel = OPSD->hasCancel();
1381 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1382 HasCancel = OPFD->hasCancel();
1383 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1384 HasCancel = OPFD->hasCancel();
1385 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1386 HasCancel = OPFD->hasCancel();
1387 else if (const auto *OPFD =
1388 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1389 HasCancel = OPFD->hasCancel();
1390 else if (const auto *OPFD =
1391 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1392 HasCancel = OPFD->hasCancel();
1393 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1394 HasCancel, OutlinedHelperName);
1395 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1396 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1397}
1398
1399llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1400 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1401 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1402 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1403 return emitParallelOrTeamsOutlinedFunction(
1404 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1405}
1406
1407llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1408 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1409 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1410 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1411 return emitParallelOrTeamsOutlinedFunction(
1412 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1413}
1414
1415llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1416 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1417 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1418 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1419 bool Tied, unsigned &NumberOfParts) {
1420 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1421 PrePostActionTy &) {
1422 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1423 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1424 llvm::Value *TaskArgs[] = {
1425 UpLoc, ThreadID,
1426 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1427 TaskTVar->getType()->castAs<PointerType>())
1428 .getPointer()};
1429 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1430 };
1431 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1432 UntiedCodeGen);
1433 CodeGen.setAction(Action);
1434 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1435, __PRETTY_FUNCTION__))
1435 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1435, __PRETTY_FUNCTION__))
;
1436 const OpenMPDirectiveKind Region =
1437 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1438 : OMPD_task;
1439 const CapturedStmt *CS = D.getCapturedStmt(Region);
1440 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1441 CodeGenFunction CGF(CGM, true);
1442 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1443 InnermostKind,
1444 TD ? TD->hasCancel() : false, Action);
1445 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1446 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1447 if (!Tied)
1448 NumberOfParts = Action.getNumberOfParts();
1449 return Res;
1450}
1451
1452static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1453 const RecordDecl *RD, const CGRecordLayout &RL,
1454 ArrayRef<llvm::Constant *> Data) {
1455 llvm::StructType *StructTy = RL.getLLVMType();
1456 unsigned PrevIdx = 0;
1457 ConstantInitBuilder CIBuilder(CGM);
1458 auto DI = Data.begin();
1459 for (const FieldDecl *FD : RD->fields()) {
1460 unsigned Idx = RL.getLLVMFieldNo(FD);
1461 // Fill the alignment.
1462 for (unsigned I = PrevIdx; I < Idx; ++I)
1463 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1464 PrevIdx = Idx + 1;
1465 Fields.add(*DI);
1466 ++DI;
1467 }
1468}
1469
1470template <class... As>
1471static llvm::GlobalVariable *
1472createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1473 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1474 As &&... Args) {
1475 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1476 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1477 ConstantInitBuilder CIBuilder(CGM);
1478 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1479 buildStructValue(Fields, CGM, RD, RL, Data);
1480 return Fields.finishAndCreateGlobal(
1481 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1482 std::forward<As>(Args)...);
1483}
1484
1485template <typename T>
1486static void
1487createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1488 ArrayRef<llvm::Constant *> Data,
1489 T &Parent) {
1490 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1491 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1492 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1493 buildStructValue(Fields, CGM, RD, RL, Data);
1494 Fields.finishAndAddTo(Parent);
1495}
1496
1497Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1498 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1499 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1500 FlagsTy FlagsKey(Flags, Reserved2Flags);
1501 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1502 if (!Entry) {
1503 if (!DefaultOpenMPPSource) {
1504 // Initialize default location for psource field of ident_t structure of
1505 // all ident_t objects. Format is ";file;function;line;column;;".
1506 // Taken from
1507 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1508 DefaultOpenMPPSource =
1509 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1510 DefaultOpenMPPSource =
1511 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1512 }
1513
1514 llvm::Constant *Data[] = {
1515 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1516 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1517 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1518 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1519 llvm::GlobalValue *DefaultOpenMPLocation =
1520 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1521 llvm::GlobalValue::PrivateLinkage);
1522 DefaultOpenMPLocation->setUnnamedAddr(
1523 llvm::GlobalValue::UnnamedAddr::Global);
1524
1525 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1526 }
1527 return Address(Entry, Align);
1528}
1529
1530void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1531 bool AtCurrentPoint) {
1532 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1533 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1533, __PRETTY_FUNCTION__))
;
1534
1535 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1536 if (AtCurrentPoint) {
1537 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1538 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1539 } else {
1540 Elem.second.ServiceInsertPt =
1541 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1542 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1543 }
1544}
1545
1546void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1547 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1548 if (Elem.second.ServiceInsertPt) {
1549 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1550 Elem.second.ServiceInsertPt = nullptr;
1551 Ptr->eraseFromParent();
1552 }
1553}
1554
1555llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1556 SourceLocation Loc,
1557 unsigned Flags) {
1558 Flags |= OMP_IDENT_KMPC;
1559 // If no debug info is generated - return global default location.
1560 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1561 Loc.isInvalid())
1562 return getOrCreateDefaultLocation(Flags).getPointer();
1563
1564 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1564, __PRETTY_FUNCTION__))
;
1565
1566 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1567 Address LocValue = Address::invalid();
1568 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1569 if (I != OpenMPLocThreadIDMap.end())
1570 LocValue = Address(I->second.DebugLoc, Align);
1571
1572 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1573 // GetOpenMPThreadID was called before this routine.
1574 if (!LocValue.isValid()) {
1575 // Generate "ident_t .kmpc_loc.addr;"
1576 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1577 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1578 Elem.second.DebugLoc = AI.getPointer();
1579 LocValue = AI;
1580
1581 if (!Elem.second.ServiceInsertPt)
1582 setLocThreadIdInsertPt(CGF);
1583 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1584 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1585 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1586 CGF.getTypeSize(IdentQTy));
1587 }
1588
1589 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1590 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1591 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1592 LValue PSource =
1593 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1594
1595 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1596 if (OMPDebugLoc == nullptr) {
1597 SmallString<128> Buffer2;
1598 llvm::raw_svector_ostream OS2(Buffer2);
1599 // Build debug location
1600 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1601 OS2 << ";" << PLoc.getFilename() << ";";
1602 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1603 OS2 << FD->getQualifiedNameAsString();
1604 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1605 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1606 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1607 }
1608 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1609 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1610
1611 // Our callers always pass this to a runtime function, so for
1612 // convenience, go ahead and return a naked pointer.
1613 return LocValue.getPointer();
1614}
1615
1616llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1617 SourceLocation Loc) {
1618 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1618, __PRETTY_FUNCTION__))
;
1619
1620 llvm::Value *ThreadID = nullptr;
1621 // Check whether we've already cached a load of the thread id in this
1622 // function.
1623 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1624 if (I != OpenMPLocThreadIDMap.end()) {
1625 ThreadID = I->second.ThreadID;
1626 if (ThreadID != nullptr)
1627 return ThreadID;
1628 }
1629 // If exceptions are enabled, do not use parameter to avoid possible crash.
1630 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1631 !CGF.getLangOpts().CXXExceptions ||
1632 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1633 if (auto *OMPRegionInfo =
1634 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1635 if (OMPRegionInfo->getThreadIDVariable()) {
1636 // Check if this an outlined function with thread id passed as argument.
1637 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1638 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1639 // If value loaded in entry block, cache it and use it everywhere in
1640 // function.
1641 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1642 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1643 Elem.second.ThreadID = ThreadID;
1644 }
1645 return ThreadID;
1646 }
1647 }
1648 }
1649
1650 // This is not an outlined function region - need to call __kmpc_int32
1651 // kmpc_global_thread_num(ident_t *loc).
1652 // Generate thread id value and cache this value for use across the
1653 // function.
1654 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1655 if (!Elem.second.ServiceInsertPt)
1656 setLocThreadIdInsertPt(CGF);
1657 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1658 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1659 llvm::CallInst *Call = CGF.Builder.CreateCall(
1660 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1661 emitUpdateLocation(CGF, Loc));
1662 Call->setCallingConv(CGF.getRuntimeCC());
1663 Elem.second.ThreadID = Call;
1664 return Call;
1665}
1666
1667void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1668 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1668, __PRETTY_FUNCTION__))
;
1669 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1670 clearLocThreadIdInsertPt(CGF);
1671 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1672 }
1673 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1674 for(auto *D : FunctionUDRMap[CGF.CurFn])
1675 UDRMap.erase(D);
1676 FunctionUDRMap.erase(CGF.CurFn);
1677 }
1678}
1679
1680llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1681 return IdentTy->getPointerTo();
1682}
1683
1684llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1685 if (!Kmpc_MicroTy) {
1686 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1687 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1688 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1689 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1690 }
1691 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1692}
1693
1694llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1695 llvm::FunctionCallee RTLFn = nullptr;
1696 switch (static_cast<OpenMPRTLFunction>(Function)) {
1697 case OMPRTL__kmpc_fork_call: {
1698 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1699 // microtask, ...);
1700 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1701 getKmpc_MicroPointerTy()};
1702 auto *FnTy =
1703 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1704 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1705 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1706 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1707 llvm::LLVMContext &Ctx = F->getContext();
1708 llvm::MDBuilder MDB(Ctx);
1709 // Annotate the callback behavior of the __kmpc_fork_call:
1710 // - The callback callee is argument number 2 (microtask).
1711 // - The first two arguments of the callback callee are unknown (-1).
1712 // - All variadic arguments to the __kmpc_fork_call are passed to the
1713 // callback callee.
1714 F->addMetadata(
1715 llvm::LLVMContext::MD_callback,
1716 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1717 2, {-1, -1},
1718 /* VarArgsArePassed */ true)}));
1719 }
1720 }
1721 break;
1722 }
1723 case OMPRTL__kmpc_global_thread_num: {
1724 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1725 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1726 auto *FnTy =
1727 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1728 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1729 break;
1730 }
1731 case OMPRTL__kmpc_threadprivate_cached: {
1732 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1733 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1734 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1735 CGM.VoidPtrTy, CGM.SizeTy,
1736 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1737 auto *FnTy =
1738 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1739 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1740 break;
1741 }
1742 case OMPRTL__kmpc_critical: {
1743 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1744 // kmp_critical_name *crit);
1745 llvm::Type *TypeParams[] = {
1746 getIdentTyPointerTy(), CGM.Int32Ty,
1747 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1748 auto *FnTy =
1749 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1750 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1751 break;
1752 }
1753 case OMPRTL__kmpc_critical_with_hint: {
1754 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1755 // kmp_critical_name *crit, uintptr_t hint);
1756 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1757 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1758 CGM.IntPtrTy};
1759 auto *FnTy =
1760 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1761 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1762 break;
1763 }
1764 case OMPRTL__kmpc_threadprivate_register: {
1765 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1766 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1767 // typedef void *(*kmpc_ctor)(void *);
1768 auto *KmpcCtorTy =
1769 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1770 /*isVarArg*/ false)->getPointerTo();
1771 // typedef void *(*kmpc_cctor)(void *, void *);
1772 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1773 auto *KmpcCopyCtorTy =
1774 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1775 /*isVarArg*/ false)
1776 ->getPointerTo();
1777 // typedef void (*kmpc_dtor)(void *);
1778 auto *KmpcDtorTy =
1779 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1780 ->getPointerTo();
1781 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1782 KmpcCopyCtorTy, KmpcDtorTy};
1783 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1784 /*isVarArg*/ false);
1785 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1786 break;
1787 }
1788 case OMPRTL__kmpc_end_critical: {
1789 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1790 // kmp_critical_name *crit);
1791 llvm::Type *TypeParams[] = {
1792 getIdentTyPointerTy(), CGM.Int32Ty,
1793 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1794 auto *FnTy =
1795 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1796 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1797 break;
1798 }
1799 case OMPRTL__kmpc_cancel_barrier: {
1800 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1801 // global_tid);
1802 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1803 auto *FnTy =
1804 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1805 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1806 break;
1807 }
1808 case OMPRTL__kmpc_barrier: {
1809 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1810 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1811 auto *FnTy =
1812 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1813 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1814 break;
1815 }
1816 case OMPRTL__kmpc_for_static_fini: {
1817 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1818 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1819 auto *FnTy =
1820 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1821 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1822 break;
1823 }
1824 case OMPRTL__kmpc_push_num_threads: {
1825 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1826 // kmp_int32 num_threads)
1827 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1828 CGM.Int32Ty};
1829 auto *FnTy =
1830 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1831 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1832 break;
1833 }
1834 case OMPRTL__kmpc_serialized_parallel: {
1835 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1836 // global_tid);
1837 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1838 auto *FnTy =
1839 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1840 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1841 break;
1842 }
1843 case OMPRTL__kmpc_end_serialized_parallel: {
1844 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1845 // global_tid);
1846 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1847 auto *FnTy =
1848 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1849 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1850 break;
1851 }
1852 case OMPRTL__kmpc_flush: {
1853 // Build void __kmpc_flush(ident_t *loc);
1854 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1855 auto *FnTy =
1856 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1857 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1858 break;
1859 }
1860 case OMPRTL__kmpc_master: {
1861 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1862 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1863 auto *FnTy =
1864 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1865 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1866 break;
1867 }
1868 case OMPRTL__kmpc_end_master: {
1869 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1870 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1871 auto *FnTy =
1872 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1873 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1874 break;
1875 }
1876 case OMPRTL__kmpc_omp_taskyield: {
1877 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1878 // int end_part);
1879 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1880 auto *FnTy =
1881 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1882 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1883 break;
1884 }
1885 case OMPRTL__kmpc_single: {
1886 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1887 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1888 auto *FnTy =
1889 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1890 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1891 break;
1892 }
1893 case OMPRTL__kmpc_end_single: {
1894 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1895 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1896 auto *FnTy =
1897 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1898 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1899 break;
1900 }
1901 case OMPRTL__kmpc_omp_task_alloc: {
1902 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1903 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1904 // kmp_routine_entry_t *task_entry);
1905 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1906, __PRETTY_FUNCTION__))
1906 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1906, __PRETTY_FUNCTION__))
;
1907 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1908 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1909 // Return void * and then cast to particular kmp_task_t type.
1910 auto *FnTy =
1911 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1912 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1913 break;
1914 }
1915 case OMPRTL__kmpc_omp_task: {
1916 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1917 // *new_task);
1918 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1919 CGM.VoidPtrTy};
1920 auto *FnTy =
1921 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1922 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
1923 break;
1924 }
1925 case OMPRTL__kmpc_copyprivate: {
1926 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
1927 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
1928 // kmp_int32 didit);
1929 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1930 auto *CpyFnTy =
1931 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
1932 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
1933 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1934 CGM.Int32Ty};
1935 auto *FnTy =
1936 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1937 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
1938 break;
1939 }
1940 case OMPRTL__kmpc_reduce: {
1941 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
1942 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
1943 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
1944 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1945 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1946 /*isVarArg=*/false);
1947 llvm::Type *TypeParams[] = {
1948 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1949 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1950 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1951 auto *FnTy =
1952 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1953 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
1954 break;
1955 }
1956 case OMPRTL__kmpc_reduce_nowait: {
1957 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
1958 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
1959 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
1960 // *lck);
1961 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1962 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1963 /*isVarArg=*/false);
1964 llvm::Type *TypeParams[] = {
1965 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1966 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1967 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1968 auto *FnTy =
1969 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1970 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
1971 break;
1972 }
1973 case OMPRTL__kmpc_end_reduce: {
1974 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
1975 // kmp_critical_name *lck);
1976 llvm::Type *TypeParams[] = {
1977 getIdentTyPointerTy(), CGM.Int32Ty,
1978 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1979 auto *FnTy =
1980 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1981 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
1982 break;
1983 }
1984 case OMPRTL__kmpc_end_reduce_nowait: {
1985 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
1986 // kmp_critical_name *lck);
1987 llvm::Type *TypeParams[] = {
1988 getIdentTyPointerTy(), CGM.Int32Ty,
1989 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1990 auto *FnTy =
1991 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1992 RTLFn =
1993 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
1994 break;
1995 }
1996 case OMPRTL__kmpc_omp_task_begin_if0: {
1997 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1998 // *new_task);
1999 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2000 CGM.VoidPtrTy};
2001 auto *FnTy =
2002 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2003 RTLFn =
2004 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2005 break;
2006 }
2007 case OMPRTL__kmpc_omp_task_complete_if0: {
2008 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2009 // *new_task);
2010 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2011 CGM.VoidPtrTy};
2012 auto *FnTy =
2013 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2014 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2015 /*Name=*/"__kmpc_omp_task_complete_if0");
2016 break;
2017 }
2018 case OMPRTL__kmpc_ordered: {
2019 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2020 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2021 auto *FnTy =
2022 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2023 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2024 break;
2025 }
2026 case OMPRTL__kmpc_end_ordered: {
2027 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2028 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2029 auto *FnTy =
2030 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2031 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2032 break;
2033 }
2034 case OMPRTL__kmpc_omp_taskwait: {
2035 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2036 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2037 auto *FnTy =
2038 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2039 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2040 break;
2041 }
2042 case OMPRTL__kmpc_taskgroup: {
2043 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2044 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2045 auto *FnTy =
2046 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2047 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2048 break;
2049 }
2050 case OMPRTL__kmpc_end_taskgroup: {
2051 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2052 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2053 auto *FnTy =
2054 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2055 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2056 break;
2057 }
2058 case OMPRTL__kmpc_push_proc_bind: {
2059 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2060 // int proc_bind)
2061 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2062 auto *FnTy =
2063 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2064 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2065 break;
2066 }
2067 case OMPRTL__kmpc_omp_task_with_deps: {
2068 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2069 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2070 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2071 llvm::Type *TypeParams[] = {
2072 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2073 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2074 auto *FnTy =
2075 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2076 RTLFn =
2077 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2078 break;
2079 }
2080 case OMPRTL__kmpc_omp_wait_deps: {
2081 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2082 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2083 // kmp_depend_info_t *noalias_dep_list);
2084 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2085 CGM.Int32Ty, CGM.VoidPtrTy,
2086 CGM.Int32Ty, CGM.VoidPtrTy};
2087 auto *FnTy =
2088 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2089 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2090 break;
2091 }
2092 case OMPRTL__kmpc_cancellationpoint: {
2093 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2094 // global_tid, kmp_int32 cncl_kind)
2095 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2096 auto *FnTy =
2097 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2098 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2099 break;
2100 }
2101 case OMPRTL__kmpc_cancel: {
2102 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2103 // kmp_int32 cncl_kind)
2104 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2105 auto *FnTy =
2106 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2107 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2108 break;
2109 }
2110 case OMPRTL__kmpc_push_num_teams: {
2111 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2112 // kmp_int32 num_teams, kmp_int32 num_threads)
2113 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2114 CGM.Int32Ty};
2115 auto *FnTy =
2116 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2117 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2118 break;
2119 }
2120 case OMPRTL__kmpc_fork_teams: {
2121 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2122 // microtask, ...);
2123 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2124 getKmpc_MicroPointerTy()};
2125 auto *FnTy =
2126 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2127 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2128 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2129 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2130 llvm::LLVMContext &Ctx = F->getContext();
2131 llvm::MDBuilder MDB(Ctx);
2132 // Annotate the callback behavior of the __kmpc_fork_teams:
2133 // - The callback callee is argument number 2 (microtask).
2134 // - The first two arguments of the callback callee are unknown (-1).
2135 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2136 // callback callee.
2137 F->addMetadata(
2138 llvm::LLVMContext::MD_callback,
2139 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2140 2, {-1, -1},
2141 /* VarArgsArePassed */ true)}));
2142 }
2143 }
2144 break;
2145 }
2146 case OMPRTL__kmpc_taskloop: {
2147 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2148 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2149 // sched, kmp_uint64 grainsize, void *task_dup);
2150 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2151 CGM.IntTy,
2152 CGM.VoidPtrTy,
2153 CGM.IntTy,
2154 CGM.Int64Ty->getPointerTo(),
2155 CGM.Int64Ty->getPointerTo(),
2156 CGM.Int64Ty,
2157 CGM.IntTy,
2158 CGM.IntTy,
2159 CGM.Int64Ty,
2160 CGM.VoidPtrTy};
2161 auto *FnTy =
2162 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2163 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2164 break;
2165 }
2166 case OMPRTL__kmpc_doacross_init: {
2167 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2168 // num_dims, struct kmp_dim *dims);
2169 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2170 CGM.Int32Ty,
2171 CGM.Int32Ty,
2172 CGM.VoidPtrTy};
2173 auto *FnTy =
2174 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2175 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2176 break;
2177 }
2178 case OMPRTL__kmpc_doacross_fini: {
2179 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2180 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2181 auto *FnTy =
2182 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2183 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2184 break;
2185 }
2186 case OMPRTL__kmpc_doacross_post: {
2187 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2188 // *vec);
2189 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2190 CGM.Int64Ty->getPointerTo()};
2191 auto *FnTy =
2192 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2193 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2194 break;
2195 }
2196 case OMPRTL__kmpc_doacross_wait: {
2197 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2198 // *vec);
2199 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2200 CGM.Int64Ty->getPointerTo()};
2201 auto *FnTy =
2202 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2203 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2204 break;
2205 }
2206 case OMPRTL__kmpc_task_reduction_init: {
2207 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2208 // *data);
2209 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2210 auto *FnTy =
2211 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2212 RTLFn =
2213 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2214 break;
2215 }
2216 case OMPRTL__kmpc_task_reduction_get_th_data: {
2217 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2218 // *d);
2219 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2220 auto *FnTy =
2221 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2222 RTLFn = CGM.CreateRuntimeFunction(
2223 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2224 break;
2225 }
2226 case OMPRTL__kmpc_alloc: {
2227 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2228 // al); omp_allocator_handle_t type is void *.
2229 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2230 auto *FnTy =
2231 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2232 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2233 break;
2234 }
2235 case OMPRTL__kmpc_free: {
2236 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2237 // al); omp_allocator_handle_t type is void *.
2238 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2239 auto *FnTy =
2240 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2241 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2242 break;
2243 }
2244 case OMPRTL__kmpc_push_target_tripcount: {
2245 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2246 // size);
2247 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2248 llvm::FunctionType *FnTy =
2249 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2250 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2251 break;
2252 }
2253 case OMPRTL__tgt_target: {
2254 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2255 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2256 // *arg_types);
2257 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2258 CGM.VoidPtrTy,
2259 CGM.Int32Ty,
2260 CGM.VoidPtrPtrTy,
2261 CGM.VoidPtrPtrTy,
2262 CGM.SizeTy->getPointerTo(),
2263 CGM.Int64Ty->getPointerTo()};
2264 auto *FnTy =
2265 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2266 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2267 break;
2268 }
2269 case OMPRTL__tgt_target_nowait: {
2270 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2271 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2272 // int64_t *arg_types);
2273 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2274 CGM.VoidPtrTy,
2275 CGM.Int32Ty,
2276 CGM.VoidPtrPtrTy,
2277 CGM.VoidPtrPtrTy,
2278 CGM.SizeTy->getPointerTo(),
2279 CGM.Int64Ty->getPointerTo()};
2280 auto *FnTy =
2281 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2282 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2283 break;
2284 }
2285 case OMPRTL__tgt_target_teams: {
2286 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2287 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2288 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2289 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2290 CGM.VoidPtrTy,
2291 CGM.Int32Ty,
2292 CGM.VoidPtrPtrTy,
2293 CGM.VoidPtrPtrTy,
2294 CGM.SizeTy->getPointerTo(),
2295 CGM.Int64Ty->getPointerTo(),
2296 CGM.Int32Ty,
2297 CGM.Int32Ty};
2298 auto *FnTy =
2299 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2300 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2301 break;
2302 }
2303 case OMPRTL__tgt_target_teams_nowait: {
2304 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2305 // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
2306 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2307 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2308 CGM.VoidPtrTy,
2309 CGM.Int32Ty,
2310 CGM.VoidPtrPtrTy,
2311 CGM.VoidPtrPtrTy,
2312 CGM.SizeTy->getPointerTo(),
2313 CGM.Int64Ty->getPointerTo(),
2314 CGM.Int32Ty,
2315 CGM.Int32Ty};
2316 auto *FnTy =
2317 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2318 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2319 break;
2320 }
2321 case OMPRTL__tgt_register_requires: {
2322 // Build void __tgt_register_requires(int64_t flags);
2323 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2324 auto *FnTy =
2325 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2326 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2327 break;
2328 }
2329 case OMPRTL__tgt_register_lib: {
2330 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2331 QualType ParamTy =
2332 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2333 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2334 auto *FnTy =
2335 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2336 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2337 break;
2338 }
2339 case OMPRTL__tgt_unregister_lib: {
2340 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2341 QualType ParamTy =
2342 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2343 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2344 auto *FnTy =
2345 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2346 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2347 break;
2348 }
2349 case OMPRTL__tgt_target_data_begin: {
2350 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2351 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2352 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2353 CGM.Int32Ty,
2354 CGM.VoidPtrPtrTy,
2355 CGM.VoidPtrPtrTy,
2356 CGM.SizeTy->getPointerTo(),
2357 CGM.Int64Ty->getPointerTo()};
2358 auto *FnTy =
2359 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2360 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2361 break;
2362 }
2363 case OMPRTL__tgt_target_data_begin_nowait: {
2364 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2365 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2366 // *arg_types);
2367 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2368 CGM.Int32Ty,
2369 CGM.VoidPtrPtrTy,
2370 CGM.VoidPtrPtrTy,
2371 CGM.SizeTy->getPointerTo(),
2372 CGM.Int64Ty->getPointerTo()};
2373 auto *FnTy =
2374 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2375 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2376 break;
2377 }
2378 case OMPRTL__tgt_target_data_end: {
2379 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2380 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2381 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2382 CGM.Int32Ty,
2383 CGM.VoidPtrPtrTy,
2384 CGM.VoidPtrPtrTy,
2385 CGM.SizeTy->getPointerTo(),
2386 CGM.Int64Ty->getPointerTo()};
2387 auto *FnTy =
2388 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2389 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2390 break;
2391 }
2392 case OMPRTL__tgt_target_data_end_nowait: {
2393 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2394 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2395 // *arg_types);
2396 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2397 CGM.Int32Ty,
2398 CGM.VoidPtrPtrTy,
2399 CGM.VoidPtrPtrTy,
2400 CGM.SizeTy->getPointerTo(),
2401 CGM.Int64Ty->getPointerTo()};
2402 auto *FnTy =
2403 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2404 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2405 break;
2406 }
2407 case OMPRTL__tgt_target_data_update: {
2408 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2409 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2410 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2411 CGM.Int32Ty,
2412 CGM.VoidPtrPtrTy,
2413 CGM.VoidPtrPtrTy,
2414 CGM.SizeTy->getPointerTo(),
2415 CGM.Int64Ty->getPointerTo()};
2416 auto *FnTy =
2417 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2418 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2419 break;
2420 }
2421 case OMPRTL__tgt_target_data_update_nowait: {
2422 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2423 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2424 // *arg_types);
2425 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2426 CGM.Int32Ty,
2427 CGM.VoidPtrPtrTy,
2428 CGM.VoidPtrPtrTy,
2429 CGM.SizeTy->getPointerTo(),
2430 CGM.Int64Ty->getPointerTo()};
2431 auto *FnTy =
2432 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2433 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2434 break;
2435 }
2436 }
2437 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2437, __PRETTY_FUNCTION__))
;
2438 return RTLFn;
2439}
2440
2441llvm::FunctionCallee
2442CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2443 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2444, __PRETTY_FUNCTION__))
2444 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2444, __PRETTY_FUNCTION__))
;
2445 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2446 : "__kmpc_for_static_init_4u")
2447 : (IVSigned ? "__kmpc_for_static_init_8"
2448 : "__kmpc_for_static_init_8u");
2449 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2450 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2451 llvm::Type *TypeParams[] = {
2452 getIdentTyPointerTy(), // loc
2453 CGM.Int32Ty, // tid
2454 CGM.Int32Ty, // schedtype
2455 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2456 PtrTy, // p_lower
2457 PtrTy, // p_upper
2458 PtrTy, // p_stride
2459 ITy, // incr
2460 ITy // chunk
2461 };
2462 auto *FnTy =
2463 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2464 return CGM.CreateRuntimeFunction(FnTy, Name);
2465}
2466
2467llvm::FunctionCallee
2468CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2469 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2470, __PRETTY_FUNCTION__))
2470 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2470, __PRETTY_FUNCTION__))
;
2471 StringRef Name =
2472 IVSize == 32
2473 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2474 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2475 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2476 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2477 CGM.Int32Ty, // tid
2478 CGM.Int32Ty, // schedtype
2479 ITy, // lower
2480 ITy, // upper
2481 ITy, // stride
2482 ITy // chunk
2483 };
2484 auto *FnTy =
2485 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2486 return CGM.CreateRuntimeFunction(FnTy, Name);
2487}
2488
2489llvm::FunctionCallee
2490CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2491 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2492, __PRETTY_FUNCTION__))
2492 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2492, __PRETTY_FUNCTION__))
;
2493 StringRef Name =
2494 IVSize == 32
2495 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2496 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2497 llvm::Type *TypeParams[] = {
2498 getIdentTyPointerTy(), // loc
2499 CGM.Int32Ty, // tid
2500 };
2501 auto *FnTy =
2502 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2503 return CGM.CreateRuntimeFunction(FnTy, Name);
2504}
2505
2506llvm::FunctionCallee
2507CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2508 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2509, __PRETTY_FUNCTION__))
2509 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2509, __PRETTY_FUNCTION__))
;
2510 StringRef Name =
2511 IVSize == 32
2512 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2513 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2514 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2515 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2516 llvm::Type *TypeParams[] = {
2517 getIdentTyPointerTy(), // loc
2518 CGM.Int32Ty, // tid
2519 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2520 PtrTy, // p_lower
2521 PtrTy, // p_upper
2522 PtrTy // p_stride
2523 };
2524 auto *FnTy =
2525 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2526 return CGM.CreateRuntimeFunction(FnTy, Name);
2527}
2528
2529Address CGOpenMPRuntime::getAddrOfDeclareTargetLink(const VarDecl *VD) {
2530 if (CGM.getLangOpts().OpenMPSimd)
2531 return Address::invalid();
2532 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2533 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2534 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Link) {
2535 SmallString<64> PtrName;
2536 {
2537 llvm::raw_svector_ostream OS(PtrName);
2538 OS << CGM.getMangledName(GlobalDecl(VD)) << "_decl_tgt_link_ptr";
2539 }
2540 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2541 if (!Ptr) {
2542 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2543 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2544 PtrName);
2545 if (!CGM.getLangOpts().OpenMPIsDevice) {
2546 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2547 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2548 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2549 }
2550 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ptr));
2551 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2552 }
2553 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2554 }
2555 return Address::invalid();
2556}
2557
2558llvm::Constant *
2559CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2560 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2561, __PRETTY_FUNCTION__))
2561 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2561, __PRETTY_FUNCTION__))
;
2562 // Lookup the entry, lazily creating it if necessary.
2563 std::string Suffix = getName({"cache", ""});
2564 return getOrCreateInternalVariable(
2565 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2566}
2567
2568Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2569 const VarDecl *VD,
2570 Address VDAddr,
2571 SourceLocation Loc) {
2572 if (CGM.getLangOpts().OpenMPUseTLS &&
2573 CGM.getContext().getTargetInfo().isTLSSupported())
2574 return VDAddr;
2575
2576 llvm::Type *VarTy = VDAddr.getElementType();
2577 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2578 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2579 CGM.Int8PtrTy),
2580 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2581 getOrCreateThreadPrivateCache(VD)};
2582 return Address(CGF.EmitRuntimeCall(
2583 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2584 VDAddr.getAlignment());
2585}
2586
2587void CGOpenMPRuntime::emitThreadPrivateVarInit(
2588 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2589 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2590 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2591 // library.
2592 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2593 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2594 OMPLoc);
2595 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2596 // to register constructor/destructor for variable.
2597 llvm::Value *Args[] = {
2598 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2599 Ctor, CopyCtor, Dtor};
2600 CGF.EmitRuntimeCall(
2601 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2602}
2603
2604llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2605 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2606 bool PerformInit, CodeGenFunction *CGF) {
2607 if (CGM.getLangOpts().OpenMPUseTLS &&
2608 CGM.getContext().getTargetInfo().isTLSSupported())
2609 return nullptr;
2610
2611 VD = VD->getDefinition(CGM.getContext());
2612 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2613 QualType ASTTy = VD->getType();
2614
2615 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2616 const Expr *Init = VD->getAnyInitializer();
2617 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2618 // Generate function that re-emits the declaration's initializer into the
2619 // threadprivate copy of the variable VD
2620 CodeGenFunction CtorCGF(CGM);
2621 FunctionArgList Args;
2622 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2623 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2624 ImplicitParamDecl::Other);
2625 Args.push_back(&Dst);
2626
2627 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2628 CGM.getContext().VoidPtrTy, Args);
2629 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2630 std::string Name = getName({"__kmpc_global_ctor_", ""});
2631 llvm::Function *Fn =
2632 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2633 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2634 Args, Loc, Loc);
2635 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2636 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2637 CGM.getContext().VoidPtrTy, Dst.getLocation());
2638 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2639 Arg = CtorCGF.Builder.CreateElementBitCast(
2640 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2641 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2642 /*IsInitializer=*/true);
2643 ArgVal = CtorCGF.EmitLoadOfScalar(
2644 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2645 CGM.getContext().VoidPtrTy, Dst.getLocation());
2646 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2647 CtorCGF.FinishFunction();
2648 Ctor = Fn;
2649 }
2650 if (VD->getType().isDestructedType() != QualType::DK_none) {
2651 // Generate function that emits destructor call for the threadprivate copy
2652 // of the variable VD
2653 CodeGenFunction DtorCGF(CGM);
2654 FunctionArgList Args;
2655 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2656 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2657 ImplicitParamDecl::Other);
2658 Args.push_back(&Dst);
2659
2660 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2661 CGM.getContext().VoidTy, Args);
2662 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2663 std::string Name = getName({"__kmpc_global_dtor_", ""});
2664 llvm::Function *Fn =
2665 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2666 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2667 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2668 Loc, Loc);
2669 // Create a scope with an artificial location for the body of this function.
2670 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2671 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2672 DtorCGF.GetAddrOfLocalVar(&Dst),
2673 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2674 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2675 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2676 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2677 DtorCGF.FinishFunction();
2678 Dtor = Fn;
2679 }
2680 // Do not emit init function if it is not required.
2681 if (!Ctor && !Dtor)
2682 return nullptr;
2683
2684 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2685 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2686 /*isVarArg=*/false)
2687 ->getPointerTo();
2688 // Copying constructor for the threadprivate variable.
2689 // Must be NULL - reserved by runtime, but currently it requires that this
2690 // parameter is always NULL. Otherwise it fires assertion.
2691 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2692 if (Ctor == nullptr) {
2693 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2694 /*isVarArg=*/false)
2695 ->getPointerTo();
2696 Ctor = llvm::Constant::getNullValue(CtorTy);
2697 }
2698 if (Dtor == nullptr) {
2699 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2700 /*isVarArg=*/false)
2701 ->getPointerTo();
2702 Dtor = llvm::Constant::getNullValue(DtorTy);
2703 }
2704 if (!CGF) {
2705 auto *InitFunctionTy =
2706 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2707 std::string Name = getName({"__omp_threadprivate_init_", ""});
2708 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2709 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2710 CodeGenFunction InitCGF(CGM);
2711 FunctionArgList ArgList;
2712 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2713 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2714 Loc, Loc);
2715 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2716 InitCGF.FinishFunction();
2717 return InitFunction;
2718 }
2719 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2720 }
2721 return nullptr;
2722}
2723
2724/// Obtain information that uniquely identifies a target entry. This
2725/// consists of the file and device IDs as well as line number associated with
2726/// the relevant entry source location.
2727static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2728 unsigned &DeviceID, unsigned &FileID,
2729 unsigned &LineNum) {
2730 SourceManager &SM = C.getSourceManager();
2731
2732 // The loc should be always valid and have a file ID (the user cannot use
2733 // #pragma directives in macros)
2734
2735 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2735, __PRETTY_FUNCTION__))
;
2736
2737 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2738 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2738, __PRETTY_FUNCTION__))
;
2739
2740 llvm::sys::fs::UniqueID ID;
2741 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2742 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2743 << PLoc.getFilename() << EC.message();
2744
2745 DeviceID = ID.getDevice();
2746 FileID = ID.getFile();
2747 LineNum = PLoc.getLine();
2748}
2749
2750bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2751 llvm::GlobalVariable *Addr,
2752 bool PerformInit) {
2753 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2754 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2755 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link)
1
Assuming the condition is false
2
Taking false branch
2756 return CGM.getLangOpts().OpenMPIsDevice;
2757 VD = VD->getDefinition(CGM.getContext());
3
Calling 'VarDecl::getDefinition'
5
Returning from 'VarDecl::getDefinition'
6
Value assigned to 'VD'
2758 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
7
Assuming 'VD' is null
8
Taking false branch
2759 return CGM.getLangOpts().OpenMPIsDevice;
2760
2761 QualType ASTTy = VD->getType();
9
Called C++ object pointer is null
2762
2763 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2764 // Produce the unique prefix to identify the new target regions. We use
2765 // the source location of the variable declaration which we know to not
2766 // conflict with any target region.
2767 unsigned DeviceID;
2768 unsigned FileID;
2769 unsigned Line;
2770 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2771 SmallString<128> Buffer, Out;
2772 {
2773 llvm::raw_svector_ostream OS(Buffer);
2774 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2775 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2776 }
2777
2778 const Expr *Init = VD->getAnyInitializer();
2779 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2780 llvm::Constant *Ctor;
2781 llvm::Constant *ID;
2782 if (CGM.getLangOpts().OpenMPIsDevice) {
2783 // Generate function that re-emits the declaration's initializer into
2784 // the threadprivate copy of the variable VD
2785 CodeGenFunction CtorCGF(CGM);
2786
2787 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2788 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2789 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2790 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2791 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2792 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2793 FunctionArgList(), Loc, Loc);
2794 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2795 CtorCGF.EmitAnyExprToMem(Init,
2796 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2797 Init->getType().getQualifiers(),
2798 /*IsInitializer=*/true);
2799 CtorCGF.FinishFunction();
2800 Ctor = Fn;
2801 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2802 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2803 } else {
2804 Ctor = new llvm::GlobalVariable(
2805 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2806 llvm::GlobalValue::PrivateLinkage,
2807 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2808 ID = Ctor;
2809 }
2810
2811 // Register the information for the entry associated with the constructor.
2812 Out.clear();
2813 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2814 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2815 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2816 }
2817 if (VD->getType().isDestructedType() != QualType::DK_none) {
2818 llvm::Constant *Dtor;
2819 llvm::Constant *ID;
2820 if (CGM.getLangOpts().OpenMPIsDevice) {
2821 // Generate function that emits destructor call for the threadprivate
2822 // copy of the variable VD
2823 CodeGenFunction DtorCGF(CGM);
2824
2825 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2826 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2827 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2828 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2829 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2830 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2831 FunctionArgList(), Loc, Loc);
2832 // Create a scope with an artificial location for the body of this
2833 // function.
2834 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2835 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2836 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2837 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2838 DtorCGF.FinishFunction();
2839 Dtor = Fn;
2840 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2841 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2842 } else {
2843 Dtor = new llvm::GlobalVariable(
2844 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2845 llvm::GlobalValue::PrivateLinkage,
2846 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2847 ID = Dtor;
2848 }
2849 // Register the information for the entry associated with the destructor.
2850 Out.clear();
2851 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2852 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2853 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2854 }
2855 return CGM.getLangOpts().OpenMPIsDevice;
2856}
2857
2858Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2859 QualType VarType,
2860 StringRef Name) {
2861 std::string Suffix = getName({"artificial", ""});
2862 std::string CacheSuffix = getName({"cache", ""});
2863 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2864 llvm::Value *GAddr =
2865 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2866 llvm::Value *Args[] = {
2867 emitUpdateLocation(CGF, SourceLocation()),
2868 getThreadID(CGF, SourceLocation()),
2869 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2870 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2871 /*IsSigned=*/false),
2872 getOrCreateInternalVariable(
2873 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2874 return Address(
2875 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2876 CGF.EmitRuntimeCall(
2877 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2878 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2879 CGM.getPointerAlign());
2880}
2881
2882void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
2883 const RegionCodeGenTy &ThenGen,
2884 const RegionCodeGenTy &ElseGen) {
2885 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2886
2887 // If the condition constant folds and can be elided, try to avoid emitting
2888 // the condition and the dead arm of the if/else.
2889 bool CondConstant;
2890 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2891 if (CondConstant)
2892 ThenGen(CGF);
2893 else
2894 ElseGen(CGF);
2895 return;
2896 }
2897
2898 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2899 // emit the conditional branch.
2900 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2901 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2902 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2903 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2904
2905 // Emit the 'then' code.
2906 CGF.EmitBlock(ThenBlock);
2907 ThenGen(CGF);
2908 CGF.EmitBranch(ContBlock);
2909 // Emit the 'else' code if present.
2910 // There is no need to emit line number for unconditional branch.
2911 (void)ApplyDebugLocation::CreateEmpty(CGF);
2912 CGF.EmitBlock(ElseBlock);
2913 ElseGen(CGF);
2914 // There is no need to emit line number for unconditional branch.
2915 (void)ApplyDebugLocation::CreateEmpty(CGF);
2916 CGF.EmitBranch(ContBlock);
2917 // Emit the continuation block for code after the if.
2918 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2919}
2920
2921void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2922 llvm::Function *OutlinedFn,
2923 ArrayRef<llvm::Value *> CapturedVars,
2924 const Expr *IfCond) {
2925 if (!CGF.HaveInsertPoint())
2926 return;
2927 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2928 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
2929 PrePostActionTy &) {
2930 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2931 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2932 llvm::Value *Args[] = {
2933 RTLoc,
2934 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2935 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2936 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2937 RealArgs.append(std::begin(Args), std::end(Args));
2938 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2939
2940 llvm::FunctionCallee RTLFn =
2941 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
2942 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2943 };
2944 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
2945 PrePostActionTy &) {
2946 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2947 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2948 // Build calls:
2949 // __kmpc_serialized_parallel(&Loc, GTid);
2950 llvm::Value *Args[] = {RTLoc, ThreadID};
2951 CGF.EmitRuntimeCall(
2952 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
2953
2954 // OutlinedFn(&GTid, &zero, CapturedStruct);
2955 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2956 /*Name*/ ".zero.addr");
2957 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2958 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2959 // ThreadId for serialized parallels is 0.
2960 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2961 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2962 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2963 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2964
2965 // __kmpc_end_serialized_parallel(&Loc, GTid);
2966 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2967 CGF.EmitRuntimeCall(
2968 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
2969 EndArgs);
2970 };
2971 if (IfCond) {
2972 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
2973 } else {
2974 RegionCodeGenTy ThenRCG(ThenGen);
2975 ThenRCG(CGF);
2976 }
2977}
2978
2979// If we're inside an (outlined) parallel region, use the region info's
2980// thread-ID variable (it is passed in a first argument of the outlined function
2981// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2982// regular serial code region, get thread ID by calling kmp_int32
2983// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2984// return the address of that temp.
2985Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2986 SourceLocation Loc) {
2987 if (auto *OMPRegionInfo =
2988 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2989 if (OMPRegionInfo->getThreadIDVariable())
2990 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2991
2992 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2993 QualType Int32Ty =
2994 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2995 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2996 CGF.EmitStoreOfScalar(ThreadID,
2997 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2998
2999 return ThreadIDTemp;
3000}
3001
3002llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3003 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3004 SmallString<256> Buffer;
3005 llvm::raw_svector_ostream Out(Buffer);
3006 Out << Name;
3007 StringRef RuntimeName = Out.str();
3008 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3009 if (Elem.second) {
3010 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3011, __PRETTY_FUNCTION__))
3011 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3011, __PRETTY_FUNCTION__))
;
3012 return &*Elem.second;
3013 }
3014
3015 return Elem.second = new llvm::GlobalVariable(
3016 CGM.getModule(), Ty, /*IsConstant*/ false,
3017 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3018 Elem.first(), /*InsertBefore=*/nullptr,
3019 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3020}
3021
3022llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3023 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3024 std::string Name = getName({Prefix, "var"});
3025 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3026}
3027
3028namespace {
3029/// Common pre(post)-action for different OpenMP constructs.
3030class CommonActionTy final : public PrePostActionTy {
3031 llvm::FunctionCallee EnterCallee;
3032 ArrayRef<llvm::Value *> EnterArgs;
3033 llvm::FunctionCallee ExitCallee;
3034 ArrayRef<llvm::Value *> ExitArgs;
3035 bool Conditional;
3036 llvm::BasicBlock *ContBlock = nullptr;
3037
3038public:
3039 CommonActionTy(llvm::FunctionCallee EnterCallee,
3040 ArrayRef<llvm::Value *> EnterArgs,
3041 llvm::FunctionCallee ExitCallee,
3042 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3043 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3044 ExitArgs(ExitArgs), Conditional(Conditional) {}
3045 void Enter(CodeGenFunction &CGF) override {
3046 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3047 if (Conditional) {
3048 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3049 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3050 ContBlock = CGF.createBasicBlock("omp_if.end");
3051 // Generate the branch (If-stmt)
3052 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3053 CGF.EmitBlock(ThenBlock);
3054 }
3055 }
3056 void Done(CodeGenFunction &CGF) {
3057 // Emit the rest of blocks/branches
3058 CGF.EmitBranch(ContBlock);
3059 CGF.EmitBlock(ContBlock, true);
3060 }
3061 void Exit(CodeGenFunction &CGF) override {
3062 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3063 }
3064};
3065} // anonymous namespace
3066
3067void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3068 StringRef CriticalName,
3069 const RegionCodeGenTy &CriticalOpGen,
3070 SourceLocation Loc, const Expr *Hint) {
3071 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3072 // CriticalOpGen();
3073 // __kmpc_end_critical(ident_t *, gtid, Lock);
3074 // Prepare arguments and build a call to __kmpc_critical
3075 if (!CGF.HaveInsertPoint())
3076 return;
3077 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3078 getCriticalRegionLock(CriticalName)};
3079 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3080 std::end(Args));
3081 if (Hint) {
3082 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3083 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3084 }
3085 CommonActionTy Action(
3086 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3087 : OMPRTL__kmpc_critical),
3088 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3089 CriticalOpGen.setAction(Action);
3090 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3091}
3092
3093void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3094 const RegionCodeGenTy &MasterOpGen,
3095 SourceLocation Loc) {
3096 if (!CGF.HaveInsertPoint())
3097 return;
3098 // if(__kmpc_master(ident_t *, gtid)) {
3099 // MasterOpGen();
3100 // __kmpc_end_master(ident_t *, gtid);
3101 // }
3102 // Prepare arguments and build a call to __kmpc_master
3103 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3104 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3105 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3106 /*Conditional=*/true);
3107 MasterOpGen.setAction(Action);
3108 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3109 Action.Done(CGF);
3110}
3111
3112void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3113 SourceLocation Loc) {
3114 if (!CGF.HaveInsertPoint())
3115 return;
3116 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3117 llvm::Value *Args[] = {
3118 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3119 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3120 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3121 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3122 Region->emitUntiedSwitch(CGF);
3123}
3124
3125void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3126 const RegionCodeGenTy &TaskgroupOpGen,
3127 SourceLocation Loc) {
3128 if (!CGF.HaveInsertPoint())
3129 return;
3130 // __kmpc_taskgroup(ident_t *, gtid);
3131 // TaskgroupOpGen();
3132 // __kmpc_end_taskgroup(ident_t *, gtid);
3133 // Prepare arguments and build a call to __kmpc_taskgroup
3134 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3135 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3136 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3137 Args);
3138 TaskgroupOpGen.setAction(Action);
3139 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3140}
3141
3142/// Given an array of pointers to variables, project the address of a
3143/// given variable.
3144static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3145 unsigned Index, const VarDecl *Var) {
3146 // Pull out the pointer to the variable.
3147 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3148 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3149
3150 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3151 Addr = CGF.Builder.CreateElementBitCast(
3152 Addr, CGF.ConvertTypeForMem(Var->getType()));
3153 return Addr;
3154}
3155
3156static llvm::Value *emitCopyprivateCopyFunction(
3157 CodeGenModule &CGM, llvm::Type *ArgsType,
3158 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3159 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3160 SourceLocation Loc) {
3161 ASTContext &C = CGM.getContext();
3162 // void copy_func(void *LHSArg, void *RHSArg);
3163 FunctionArgList Args;
3164 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3165 ImplicitParamDecl::Other);
3166 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3167 ImplicitParamDecl::Other);
3168 Args.push_back(&LHSArg);
3169 Args.push_back(&RHSArg);
3170 const auto &CGFI =
3171 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3172 std::string Name =
3173 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3174 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3175 llvm::GlobalValue::InternalLinkage, Name,
3176 &CGM.getModule());
3177 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3178 Fn->setDoesNotRecurse();
3179 CodeGenFunction CGF(CGM);
3180 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3181 // Dest = (void*[n])(LHSArg);
3182 // Src = (void*[n])(RHSArg);
3183 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3184 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3185 ArgsType), CGF.getPointerAlign());
3186 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3187 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3188 ArgsType), CGF.getPointerAlign());
3189 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3190 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3191 // ...
3192 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3193 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3194 const auto *DestVar =
3195 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3196 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3197
3198 const auto *SrcVar =
3199 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3200 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3201
3202 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3203 QualType Type = VD->getType();
3204 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3205 }
3206 CGF.FinishFunction();
3207 return Fn;
3208}
3209
3210void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3211 const RegionCodeGenTy &SingleOpGen,
3212 SourceLocation Loc,
3213 ArrayRef<const Expr *> CopyprivateVars,
3214 ArrayRef<const Expr *> SrcExprs,
3215 ArrayRef<const Expr *> DstExprs,
3216 ArrayRef<const Expr *> AssignmentOps) {
3217 if (!CGF.HaveInsertPoint())
3218 return;
3219 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3221, __PRETTY_FUNCTION__))
3220 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3221, __PRETTY_FUNCTION__))
3221 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3221, __PRETTY_FUNCTION__))
;
3222 ASTContext &C = CGM.getContext();
3223 // int32 did_it = 0;
3224 // if(__kmpc_single(ident_t *, gtid)) {
3225 // SingleOpGen();
3226 // __kmpc_end_single(ident_t *, gtid);
3227 // did_it = 1;
3228 // }
3229 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3230 // <copy_func>, did_it);
3231
3232 Address DidIt = Address::invalid();
3233 if (!CopyprivateVars.empty()) {
3234 // int32 did_it = 0;
3235 QualType KmpInt32Ty =
3236 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3237 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3238 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3239 }
3240 // Prepare arguments and build a call to __kmpc_single
3241 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3242 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3243 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3244 /*Conditional=*/true);
3245 SingleOpGen.setAction(Action);
3246 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3247 if (DidIt.isValid()) {
3248 // did_it = 1;
3249 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3250 }
3251 Action.Done(CGF);
3252 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3253 // <copy_func>, did_it);
3254 if (DidIt.isValid()) {
3255 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3256 QualType CopyprivateArrayTy =
3257 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3258 /*IndexTypeQuals=*/0);
3259 // Create a list of all private variables for copyprivate.
3260 Address CopyprivateList =
3261 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3262 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3263 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3264 CGF.Builder.CreateStore(
3265 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3266 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3267 Elem);
3268 }
3269 // Build function that copies private values from single region to all other
3270 // threads in the corresponding parallel region.
3271 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3272 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3273 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3274 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3275 Address CL =
3276 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3277 CGF.VoidPtrTy);
3278 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3279 llvm::Value *Args[] = {
3280 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3281 getThreadID(CGF, Loc), // i32 <gtid>
3282 BufSize, // size_t <buf_size>
3283 CL.getPointer(), // void *<copyprivate list>
3284 CpyFn, // void (*) (void *, void *) <copy_func>
3285 DidItVal // i32 did_it
3286 };
3287 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3288 }
3289}
3290
3291void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3292 const RegionCodeGenTy &OrderedOpGen,
3293 SourceLocation Loc, bool IsThreads) {
3294 if (!CGF.HaveInsertPoint())
3295 return;
3296 // __kmpc_ordered(ident_t *, gtid);
3297 // OrderedOpGen();
3298 // __kmpc_end_ordered(ident_t *, gtid);
3299 // Prepare arguments and build a call to __kmpc_ordered
3300 if (IsThreads) {
3301 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3302 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3303 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3304 Args);
3305 OrderedOpGen.setAction(Action);
3306 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3307 return;
3308 }
3309 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3310}
3311
3312unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3313 unsigned Flags;
3314 if (Kind == OMPD_for)
3315 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3316 else if (Kind == OMPD_sections)
3317 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3318 else if (Kind == OMPD_single)
3319 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3320 else if (Kind == OMPD_barrier)
3321 Flags = OMP_IDENT_BARRIER_EXPL;
3322 else
3323 Flags = OMP_IDENT_BARRIER_IMPL;
3324 return Flags;
3325}
3326
3327void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3328 CodeGenFunction &CGF, const OMPLoopDirective &S,
3329 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3330 // Check if the loop directive is actually a doacross loop directive. In this
3331 // case choose static, 1 schedule.
3332 if (llvm::any_of(
3333 S.getClausesOfKind<OMPOrderedClause>(),
3334 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3335 ScheduleKind = OMPC_SCHEDULE_static;
3336 // Chunk size is 1 in this case.
3337 llvm::APInt ChunkSize(32, 1);
3338 ChunkExpr = IntegerLiteral::Create(
3339 CGF.getContext(), ChunkSize,
3340 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3341 SourceLocation());
3342 }
3343}
3344
3345void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3346 OpenMPDirectiveKind Kind, bool EmitChecks,
3347 bool ForceSimpleCall) {
3348 if (!CGF.HaveInsertPoint())
3349 return;
3350 // Build call __kmpc_cancel_barrier(loc, thread_id);
3351 // Build call __kmpc_barrier(loc, thread_id);
3352 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3353 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3354 // thread_id);
3355 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3356 getThreadID(CGF, Loc)};
3357 if (auto *OMPRegionInfo =
3358 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3359 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3360 llvm::Value *Result = CGF.EmitRuntimeCall(
3361 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3362 if (EmitChecks) {
3363 // if (__kmpc_cancel_barrier()) {
3364 // exit from construct;
3365 // }
3366 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3367 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3368 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3369 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3370 CGF.EmitBlock(ExitBB);
3371 // exit from construct;
3372 CodeGenFunction::JumpDest CancelDestination =
3373 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3374 CGF.EmitBranchThroughCleanup(CancelDestination);
3375 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3376 }
3377 return;
3378 }
3379 }
3380 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3381}
3382
3383/// Map the OpenMP loop schedule to the runtime enumeration.
3384static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3385 bool Chunked, bool Ordered) {
3386 switch (ScheduleKind) {
3387 case OMPC_SCHEDULE_static:
3388 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3389 : (Ordered ? OMP_ord_static : OMP_sch_static);
3390 case OMPC_SCHEDULE_dynamic:
3391 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3392 case OMPC_SCHEDULE_guided:
3393 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3394 case OMPC_SCHEDULE_runtime:
3395 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3396 case OMPC_SCHEDULE_auto:
3397 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3398 case OMPC_SCHEDULE_unknown:
3399 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3399, __PRETTY_FUNCTION__))
;
3400 return Ordered ? OMP_ord_static : OMP_sch_static;
3401 }
3402 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3402)
;
3403}
3404
3405/// Map the OpenMP distribute schedule to the runtime enumeration.
3406static OpenMPSchedType
3407getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3408 // only static is allowed for dist_schedule
3409 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3410}
3411
3412bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3413 bool Chunked) const {
3414 OpenMPSchedType Schedule =
3415 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3416 return Schedule == OMP_sch_static;
3417}
3418
3419bool CGOpenMPRuntime::isStaticNonchunked(
3420 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3421 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3422 return Schedule == OMP_dist_sch_static;
3423}
3424
3425bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3426 bool Chunked) const {
3427 OpenMPSchedType Schedule =
3428 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3429 return Schedule == OMP_sch_static_chunked;
3430}
3431
3432bool CGOpenMPRuntime::isStaticChunked(
3433 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3434 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3435 return Schedule == OMP_dist_sch_static_chunked;
3436}
3437
3438bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3439 OpenMPSchedType Schedule =
3440 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3441 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3441, __PRETTY_FUNCTION__))
;
3442 return Schedule != OMP_sch_static;
3443}
3444
3445static int addMonoNonMonoModifier(OpenMPSchedType Schedule,
3446 OpenMPScheduleClauseModifier M1,
3447 OpenMPScheduleClauseModifier M2) {
3448 int Modifier = 0;
3449 switch (M1) {
3450 case OMPC_SCHEDULE_MODIFIER_monotonic:
3451 Modifier = OMP_sch_modifier_monotonic;
3452 break;
3453 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3454 Modifier = OMP_sch_modifier_nonmonotonic;
3455 break;
3456 case OMPC_SCHEDULE_MODIFIER_simd:
3457 if (Schedule == OMP_sch_static_chunked)
3458 Schedule = OMP_sch_static_balanced_chunked;
3459 break;
3460 case OMPC_SCHEDULE_MODIFIER_last:
3461 case OMPC_SCHEDULE_MODIFIER_unknown:
3462 break;
3463 }
3464 switch (M2) {
3465 case OMPC_SCHEDULE_MODIFIER_monotonic:
3466 Modifier = OMP_sch_modifier_monotonic;
3467 break;
3468 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3469 Modifier = OMP_sch_modifier_nonmonotonic;
3470 break;
3471 case OMPC_SCHEDULE_MODIFIER_simd:
3472 if (Schedule == OMP_sch_static_chunked)
3473 Schedule = OMP_sch_static_balanced_chunked;
3474 break;
3475 case OMPC_SCHEDULE_MODIFIER_last:
3476 case OMPC_SCHEDULE_MODIFIER_unknown:
3477 break;
3478 }
3479 return Schedule | Modifier;
3480}
3481
3482void CGOpenMPRuntime::emitForDispatchInit(
3483 CodeGenFunction &CGF, SourceLocation Loc,
3484 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3485 bool Ordered, const DispatchRTInput &DispatchValues) {
3486 if (!CGF.HaveInsertPoint())
3487 return;
3488 OpenMPSchedType Schedule = getRuntimeSchedule(
3489 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3490 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3493, __PRETTY_FUNCTION__))
3491 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3493, __PRETTY_FUNCTION__))
3492 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3493, __PRETTY_FUNCTION__))
3493 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3493, __PRETTY_FUNCTION__))
;
3494 // Call __kmpc_dispatch_init(
3495 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3496 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3497 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3498
3499 // If the Chunk was not specified in the clause - use default value 1.
3500 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3501 : CGF.Builder.getIntN(IVSize, 1);
3502 llvm::Value *Args[] = {
3503 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3504 CGF.Builder.getInt32(addMonoNonMonoModifier(
3505 Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3506 DispatchValues.LB, // Lower
3507 DispatchValues.UB, // Upper
3508 CGF.Builder.getIntN(IVSize, 1), // Stride
3509 Chunk // Chunk
3510 };
3511 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3512}
3513
3514static void emitForStaticInitCall(
3515 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3516 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3517 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3518 const CGOpenMPRuntime::StaticRTInput &Values) {
3519 if (!CGF.HaveInsertPoint())
3520 return;
3521
3522 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3522, __PRETTY_FUNCTION__))
;
3523 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3527, __PRETTY_FUNCTION__))
3524 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3527, __PRETTY_FUNCTION__))
3525 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3527, __PRETTY_FUNCTION__))
3526 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3527, __PRETTY_FUNCTION__))
3527 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3527, __PRETTY_FUNCTION__))
;
3528
3529 // Call __kmpc_for_static_init(
3530 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3531 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3532 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3533 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3534 llvm::Value *Chunk = Values.Chunk;
3535 if (Chunk == nullptr) {
3536 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3538, __PRETTY_FUNCTION__))
3537 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3538, __PRETTY_FUNCTION__))
3538 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3538, __PRETTY_FUNCTION__))
;
3539 // If the Chunk was not specified in the clause - use default value 1.
3540 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3541 } else {
3542 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3546, __PRETTY_FUNCTION__))
3543 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3546, __PRETTY_FUNCTION__))
3544 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3546, __PRETTY_FUNCTION__))
3545 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3546, __PRETTY_FUNCTION__))
3546 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3546, __PRETTY_FUNCTION__))
;
3547 }
3548 llvm::Value *Args[] = {
3549 UpdateLocation,
3550 ThreadId,
3551 CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1,
3552 M2)), // Schedule type
3553 Values.IL.getPointer(), // &isLastIter
3554 Values.LB.getPointer(), // &LB
3555 Values.UB.getPointer(), // &UB
3556 Values.ST.getPointer(), // &Stride
3557 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3558 Chunk // Chunk
3559 };
3560 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3561}
3562
3563void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3564 SourceLocation Loc,
3565 OpenMPDirectiveKind DKind,
3566 const OpenMPScheduleTy &ScheduleKind,
3567 const StaticRTInput &Values) {
3568 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3569 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3570 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3571, __PRETTY_FUNCTION__))
3571 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3571, __PRETTY_FUNCTION__))
;
3572 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3573 isOpenMPLoopDirective(DKind)
3574 ? OMP_IDENT_WORK_LOOP
3575 : OMP_IDENT_WORK_SECTIONS);
3576 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3577 llvm::FunctionCallee StaticInitFunction =
3578 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3579 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3580 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3581}
3582
3583void CGOpenMPRuntime::emitDistributeStaticInit(
3584 CodeGenFunction &CGF, SourceLocation Loc,
3585 OpenMPDistScheduleClauseKind SchedKind,
3586 const CGOpenMPRuntime::StaticRTInput &Values) {
3587 OpenMPSchedType ScheduleNum =
3588 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3589 llvm::Value *UpdatedLocation =
3590 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3591 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3592 llvm::FunctionCallee StaticInitFunction =
3593 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3594 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3595 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3596 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3597}
3598
3599void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3600 SourceLocation Loc,
3601 OpenMPDirectiveKind DKind) {
3602 if (!CGF.HaveInsertPoint())
3603 return;
3604 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3605 llvm::Value *Args[] = {
3606 emitUpdateLocation(CGF, Loc,
3607 isOpenMPDistributeDirective(DKind)
3608 ? OMP_IDENT_WORK_DISTRIBUTE
3609 : isOpenMPLoopDirective(DKind)
3610 ? OMP_IDENT_WORK_LOOP
3611 : OMP_IDENT_WORK_SECTIONS),
3612 getThreadID(CGF, Loc)};
3613 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3614 Args);
3615}
3616
3617void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3618 SourceLocation Loc,
3619 unsigned IVSize,
3620 bool IVSigned) {
3621 if (!CGF.HaveInsertPoint())
3622 return;
3623 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3624 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3625 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3626}
3627
3628llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3629 SourceLocation Loc, unsigned IVSize,
3630 bool IVSigned, Address IL,
3631 Address LB, Address UB,
3632 Address ST) {
3633 // Call __kmpc_dispatch_next(
3634 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3635 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3636 // kmp_int[32|64] *p_stride);
3637 llvm::Value *Args[] = {
3638 emitUpdateLocation(CGF, Loc),
3639 getThreadID(CGF, Loc),
3640 IL.getPointer(), // &isLastIter
3641 LB.getPointer(), // &Lower
3642 UB.getPointer(), // &Upper
3643 ST.getPointer() // &Stride
3644 };
3645 llvm::Value *Call =
3646 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3647 return CGF.EmitScalarConversion(
3648 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3649 CGF.getContext().BoolTy, Loc);
3650}
3651
3652void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3653 llvm::Value *NumThreads,
3654 SourceLocation Loc) {
3655 if (!CGF.HaveInsertPoint())
3656 return;
3657 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3658 llvm::Value *Args[] = {
3659 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3660 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3661 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3662 Args);
3663}
3664
3665void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3666 OpenMPProcBindClauseKind ProcBind,
3667 SourceLocation Loc) {
3668 if (!CGF.HaveInsertPoint())
3669 return;
3670 // Constants for proc bind value accepted by the runtime.
3671 enum ProcBindTy {
3672 ProcBindFalse = 0,
3673 ProcBindTrue,
3674 ProcBindMaster,
3675 ProcBindClose,
3676 ProcBindSpread,
3677 ProcBindIntel,
3678 ProcBindDefault
3679 } RuntimeProcBind;
3680 switch (ProcBind) {
3681 case OMPC_PROC_BIND_master:
3682 RuntimeProcBind = ProcBindMaster;
3683 break;
3684 case OMPC_PROC_BIND_close:
3685 RuntimeProcBind = ProcBindClose;
3686 break;
3687 case OMPC_PROC_BIND_spread:
3688 RuntimeProcBind = ProcBindSpread;
3689 break;
3690 case OMPC_PROC_BIND_unknown:
3691 llvm_unreachable("Unsupported proc_bind value.")::llvm::llvm_unreachable_internal("Unsupported proc_bind value."
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3691)
;
3692 }
3693 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3694 llvm::Value *Args[] = {
3695 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3696 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3697 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3698}
3699
3700void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3701 SourceLocation Loc) {
3702 if (!CGF.HaveInsertPoint())
3703 return;
3704 // Build call void __kmpc_flush(ident_t *loc)
3705 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3706 emitUpdateLocation(CGF, Loc));
3707}
3708
3709namespace {
3710/// Indexes of fields for type kmp_task_t.
3711enum KmpTaskTFields {
3712 /// List of shared variables.
3713 KmpTaskTShareds,
3714 /// Task routine.
3715 KmpTaskTRoutine,
3716 /// Partition id for the untied tasks.
3717 KmpTaskTPartId,
3718 /// Function with call of destructors for private variables.
3719 Data1,
3720 /// Task priority.
3721 Data2,
3722 /// (Taskloops only) Lower bound.
3723 KmpTaskTLowerBound,
3724 /// (Taskloops only) Upper bound.
3725 KmpTaskTUpperBound,
3726 /// (Taskloops only) Stride.
3727 KmpTaskTStride,
3728 /// (Taskloops only) Is last iteration flag.
3729 KmpTaskTLastIter,
3730 /// (Taskloops only) Reduction data.
3731 KmpTaskTReductions,
3732};
3733} // anonymous namespace
3734
3735bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3736 return OffloadEntriesTargetRegion.empty() &&
3737 OffloadEntriesDeviceGlobalVar.empty();
3738}
3739
3740/// Initialize target region entry.
3741void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3742 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3743 StringRef ParentName, unsigned LineNum,
3744 unsigned Order) {
3745 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3747, __PRETTY_FUNCTION__))
3746 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3747, __PRETTY_FUNCTION__))
3747 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3747, __PRETTY_FUNCTION__))
;
3748 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3749 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3750 OMPTargetRegionEntryTargetRegion);
3751 ++OffloadingEntriesNum;
3752}
3753
3754void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3755 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3756 StringRef ParentName, unsigned LineNum,
3757 llvm::Constant *Addr, llvm::Constant *ID,
3758 OMPTargetRegionEntryKind Flags) {
3759 // If we are emitting code for a target, the entry is already initialized,
3760 // only has to be registered.
3761 if (CGM.getLangOpts().OpenMPIsDevice) {
3762 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3763 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3764 DiagnosticsEngine::Error,
3765 "Unable to find target region on line '%0' in the device code.");
3766 CGM.getDiags().Report(DiagID) << LineNum;
3767 return;
3768 }
3769 auto &Entry =
3770 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3771 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3771, __PRETTY_FUNCTION__))
;
3772 Entry.setAddress(Addr);
3773 Entry.setID(ID);
3774 Entry.setFlags(Flags);
3775 } else {
3776 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3777 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3778 ++OffloadingEntriesNum;
3779 }
3780}
3781
3782bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3783 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3784 unsigned LineNum) const {
3785 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3786 if (PerDevice == OffloadEntriesTargetRegion.end())
3787 return false;
3788 auto PerFile = PerDevice->second.find(FileID);
3789 if (PerFile == PerDevice->second.end())
3790 return false;
3791 auto PerParentName = PerFile->second.find(ParentName);
3792 if (PerParentName == PerFile->second.end())
3793 return false;
3794 auto PerLine = PerParentName->second.find(LineNum);
3795 if (PerLine == PerParentName->second.end())
3796 return false;
3797 // Fail if this entry is already registered.
3798 if (PerLine->second.getAddress() || PerLine->second.getID())
3799 return false;
3800 return true;
3801}
3802
3803void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3804 const OffloadTargetRegionEntryInfoActTy &Action) {
3805 // Scan all target region entries and perform the provided action.
3806 for (const auto &D : OffloadEntriesTargetRegion)
3807 for (const auto &F : D.second)
3808 for (const auto &P : F.second)
3809 for (const auto &L : P.second)
3810 Action(D.first, F.first, P.first(), L.first, L.second);
3811}
3812
3813void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3814 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3815 OMPTargetGlobalVarEntryKind Flags,
3816 unsigned Order) {
3817 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3819, __PRETTY_FUNCTION__))
3818 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3819, __PRETTY_FUNCTION__))
3819 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3819, __PRETTY_FUNCTION__))
;
3820 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3821 ++OffloadingEntriesNum;
3822}
3823
3824void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3825 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3826 CharUnits VarSize,
3827 OMPTargetGlobalVarEntryKind Flags,
3828 llvm::GlobalValue::LinkageTypes Linkage) {
3829 if (CGM.getLangOpts().OpenMPIsDevice) {
3830 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3831 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3832, __PRETTY_FUNCTION__))
3832 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3832, __PRETTY_FUNCTION__))
;
3833 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3834, __PRETTY_FUNCTION__))
3834 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3834, __PRETTY_FUNCTION__))
;
3835 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3836 if (Entry.getVarSize().isZero()) {
3837 Entry.setVarSize(VarSize);
3838 Entry.setLinkage(Linkage);
3839 }
3840 return;
3841 }
3842 Entry.setVarSize(VarSize);
3843 Entry.setLinkage(Linkage);
3844 Entry.setAddress(Addr);
3845 } else {
3846 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3847 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3848 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3849, __PRETTY_FUNCTION__))
3849 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3849, __PRETTY_FUNCTION__))
;
3850 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3851, __PRETTY_FUNCTION__))
3851 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3851, __PRETTY_FUNCTION__))
;
3852 if (Entry.getVarSize().isZero()) {
3853 Entry.setVarSize(VarSize);
3854 Entry.setLinkage(Linkage);
3855 }
3856 return;
3857 }
3858 OffloadEntriesDeviceGlobalVar.try_emplace(
3859 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3860 ++OffloadingEntriesNum;
3861 }
3862}
3863
3864void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3865 actOnDeviceGlobalVarEntriesInfo(
3866 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3867 // Scan all target region entries and perform the provided action.
3868 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3869 Action(E.getKey(), E.getValue());
3870}
3871
3872llvm::Function *
3873CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
3874 // If we don't have entries or if we are emitting code for the device, we
3875 // don't need to do anything.
3876 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
3877 return nullptr;
3878
3879 llvm::Module &M = CGM.getModule();
3880 ASTContext &C = CGM.getContext();
3881
3882 // Get list of devices we care about
3883 const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
3884
3885 // We should be creating an offloading descriptor only if there are devices
3886 // specified.
3887 assert(!Devices.empty() && "No OpenMP offloading devices??")((!Devices.empty() && "No OpenMP offloading devices??"
) ? static_cast<void> (0) : __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3887, __PRETTY_FUNCTION__))
;
3888
3889 // Create the external variables that will point to the begin and end of the
3890 // host entries section. These will be defined by the linker.
3891 llvm::Type *OffloadEntryTy =
3892 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
3893 std::string EntriesBeginName = getName({"omp_offloading", "entries_begin"});
3894 auto *HostEntriesBegin = new llvm::GlobalVariable(
3895 M, OffloadEntryTy, /*isConstant=*/true,
3896 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
3897 EntriesBeginName);
3898 std::string EntriesEndName = getName({"omp_offloading", "entries_end"});
3899 auto *HostEntriesEnd =
3900 new llvm::GlobalVariable(M, OffloadEntryTy, /*isConstant=*/true,
3901 llvm::GlobalValue::ExternalLinkage,
3902 /*Initializer=*/nullptr, EntriesEndName);
3903
3904 // Create all device images
3905 auto *DeviceImageTy = cast<llvm::StructType>(
3906 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
3907 ConstantInitBuilder DeviceImagesBuilder(CGM);
3908 ConstantArrayBuilder DeviceImagesEntries =
3909 DeviceImagesBuilder.beginArray(DeviceImageTy);
3910
3911 for (const llvm::Triple &Device : Devices) {
3912 StringRef T = Device.getTriple();
3913 std::string BeginName = getName({"omp_offloading", "img_start", ""});
3914 auto *ImgBegin = new llvm::GlobalVariable(
3915 M, CGM.Int8Ty, /*isConstant=*/true,
3916 llvm::GlobalValue::ExternalWeakLinkage,
3917 /*Initializer=*/nullptr, Twine(BeginName).concat(T));
3918 std::string EndName = getName({"omp_offloading", "img_end", ""});
3919 auto *ImgEnd = new llvm::GlobalVariable(
3920 M, CGM.Int8Ty, /*isConstant=*/true,
3921 llvm::GlobalValue::ExternalWeakLinkage,
3922 /*Initializer=*/nullptr, Twine(EndName).concat(T));
3923
3924 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
3925 HostEntriesEnd};
3926 createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
3927 DeviceImagesEntries);
3928 }
3929
3930 // Create device images global array.
3931 std::string ImagesName = getName({"omp_offloading", "device_images"});
3932 llvm::GlobalVariable *DeviceImages =
3933 DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
3934 CGM.getPointerAlign(),
3935 /*isConstant=*/true);
3936 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3937
3938 // This is a Zero array to be used in the creation of the constant expressions
3939 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3940 llvm::Constant::getNullValue(CGM.Int32Ty)};
3941
3942 // Create the target region descriptor.
3943 llvm::Constant *Data[] = {
3944 llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
3945 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3946 DeviceImages, Index),
3947 HostEntriesBegin, HostEntriesEnd};
3948 std::string Descriptor = getName({"omp_offloading", "descriptor"});
3949 llvm::GlobalVariable *Desc = createGlobalStruct(
3950 CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
3951
3952 // Emit code to register or unregister the descriptor at execution
3953 // startup or closing, respectively.
3954
3955 llvm::Function *UnRegFn;
3956 {
3957 FunctionArgList Args;
3958 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
3959 Args.push_back(&DummyPtr);
3960
3961 CodeGenFunction CGF(CGM);
3962 // Disable debug info for global (de-)initializer because they are not part
3963 // of some particular construct.
3964 CGF.disableDebugInfo();
3965 const auto &FI =
3966 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3967 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3968 std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
3969 UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
3970 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
3971 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
3972 Desc);
3973 CGF.FinishFunction();
3974 }
3975 llvm::Function *RegFn;
3976 {
3977 CodeGenFunction CGF(CGM);
3978 // Disable debug info for global (de-)initializer because they are not part
3979 // of some particular construct.
3980 CGF.disableDebugInfo();
3981 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
3982 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3983
3984 // Encode offload target triples into the registration function name. It
3985 // will serve as a comdat key for the registration/unregistration code for
3986 // this particular combination of offloading targets.
3987 SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
3988 RegFnNameParts[0] = "omp_offloading";
3989 RegFnNameParts[1] = "descriptor_reg";
3990 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
3991 [](const llvm::Triple &T) -> const std::string& {
3992 return T.getTriple();
3993 });
3994 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
3995 std::string Descriptor = getName(RegFnNameParts);
3996 RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
3997 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
3998 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
3999 // Create a variable to drive the registration and unregistration of the
4000 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
4001 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
4002 SourceLocation(), nullptr, C.CharTy,
4003 ImplicitParamDecl::Other);
4004 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
4005 CGF.FinishFunction();
4006 }
4007 if (CGM.supportsCOMDAT()) {
4008 // It is sufficient to call registration function only once, so create a
4009 // COMDAT group for registration/unregistration functions and associated
4010 // data. That would reduce startup time and code size. Registration
4011 // function serves as a COMDAT group key.
4012 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
4013 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
4014 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
4015 RegFn->setComdat(ComdatKey);
4016 UnRegFn->setComdat(ComdatKey);
4017 DeviceImages->setComdat(ComdatKey);
4018 Desc->setComdat(ComdatKey);
4019 }
4020 return RegFn;
4021}
4022
4023void CGOpenMPRuntime::createOffloadEntry(
4024 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4025 llvm::GlobalValue::LinkageTypes Linkage) {
4026 StringRef Name = Addr->getName();
4027 llvm::Module &M = CGM.getModule();
4028 llvm::LLVMContext &C = M.getContext();
4029
4030 // Create constant string with the name.
4031 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4032
4033 std::string StringName = getName({"omp_offloading", "entry_name"});
4034 auto *Str = new llvm::GlobalVariable(
4035 M, StrPtrInit->getType(), /*isConstant=*/true,
4036 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4037 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4038
4039 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4040 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4041 llvm::ConstantInt::get(CGM.SizeTy, Size),
4042 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4043 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4044 std::string EntryName = getName({"omp_offloading", "entry", ""});
4045 llvm::GlobalVariable *Entry = createGlobalStruct(
4046 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4047 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4048
4049 // The entry has to be created in the section the linker expects it to be.
4050 std::string Section = getName({"omp_offloading", "entries"});
4051 Entry->setSection(Section);
4052}
4053
4054void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4055 // Emit the offloading entries and metadata so that the device codegen side
4056 // can easily figure out what to emit. The produced metadata looks like
4057 // this:
4058 //
4059 // !omp_offload.info = !{!1, ...}
4060 //
4061 // Right now we only generate metadata for function that contain target
4062 // regions.
4063
4064 // If we do not have entries, we don't need to do anything.
4065 if (OffloadEntriesInfoManager.empty())
4066 return;
4067
4068 llvm::Module &M = CGM.getModule();
4069 llvm::LLVMContext &C = M.getContext();
4070 SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
4071 OrderedEntries(OffloadEntriesInfoManager.size());
4072 llvm::SmallVector<StringRef, 16> ParentFunctions(
4073 OffloadEntriesInfoManager.size());
4074
4075 // Auxiliary methods to create metadata values and strings.
4076 auto &&GetMDInt = [this](unsigned V) {
4077 return llvm::ConstantAsMetadata::get(
4078 llvm::ConstantInt::get(CGM.Int32Ty, V));
4079 };
4080
4081 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4082
4083 // Create the offloading info metadata node.
4084 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4085
4086 // Create function that emits metadata for each target region entry;
4087 auto &&TargetRegionMetadataEmitter =
4088 [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
4089 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4090 unsigned Line,
4091 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4092 // Generate metadata for target regions. Each entry of this metadata
4093 // contains:
4094 // - Entry 0 -> Kind of this type of metadata (0).
4095 // - Entry 1 -> Device ID of the file where the entry was identified.
4096 // - Entry 2 -> File ID of the file where the entry was identified.
4097 // - Entry 3 -> Mangled name of the function where the entry was
4098 // identified.
4099 // - Entry 4 -> Line in the file where the entry was identified.
4100 // - Entry 5 -> Order the entry was created.
4101 // The first element of the metadata node is the kind.
4102 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4103 GetMDInt(FileID), GetMDString(ParentName),
4104 GetMDInt(Line), GetMDInt(E.getOrder())};
4105
4106 // Save this entry in the right position of the ordered entries array.
4107 OrderedEntries[E.getOrder()] = &E;
4108 ParentFunctions[E.getOrder()] = ParentName;
4109
4110 // Add metadata to the named metadata node.
4111 MD->addOperand(llvm::MDNode::get(C, Ops));
4112 };
4113
4114 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4115 TargetRegionMetadataEmitter);
4116
4117 // Create function that emits metadata for each device global variable entry;
4118 auto &&DeviceGlobalVarMetadataEmitter =
4119 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4120 MD](StringRef MangledName,
4121 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4122 &E) {
4123 // Generate metadata for global variables. Each entry of this metadata
4124 // contains:
4125 // - Entry 0 -> Kind of this type of metadata (1).
4126 // - Entry 1 -> Mangled name of the variable.
4127 // - Entry 2 -> Declare target kind.
4128 // - Entry 3 -> Order the entry was created.
4129 // The first element of the metadata node is the kind.
4130 llvm::Metadata *Ops[] = {
4131 GetMDInt(E.getKind()), GetMDString(MangledName),
4132 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4133
4134 // Save this entry in the right position of the ordered entries array.
4135 OrderedEntries[E.getOrder()] = &E;
4136
4137 // Add metadata to the named metadata node.
4138 MD->addOperand(llvm::MDNode::get(C, Ops));
4139 };
4140
4141 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4142 DeviceGlobalVarMetadataEmitter);
4143
4144 for (const auto *E : OrderedEntries) {
4145 assert(E && "All ordered entries must exist!")((E && "All ordered entries must exist!") ? static_cast
<void> (0) : __assert_fail ("E && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4145, __PRETTY_FUNCTION__))
;
4146 if (const auto *CE =
4147 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4148 E)) {
4149 if (!CE->getID() || !CE->getAddress()) {
4150 // Do not blame the entry if the parent funtion is not emitted.
4151 StringRef FnName = ParentFunctions[CE->getOrder()];
4152 if (!CGM.GetGlobalValue(FnName))
4153 continue;
4154 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4155 DiagnosticsEngine::Error,
4156 "Offloading entry for target region is incorrect: either the "
4157 "address or the ID is invalid.");
4158 CGM.getDiags().Report(DiagID);
4159 continue;
4160 }
4161 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4162 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4163 } else if (const auto *CE =
4164 dyn_cast<OffloadEntriesInfoManagerTy::
4165 OffloadEntryInfoDeviceGlobalVar>(E)) {
4166 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4167 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4168 CE->getFlags());
4169 switch (Flags) {
4170 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4171 if (!CE->getAddress()) {
4172 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4173 DiagnosticsEngine::Error,
4174 "Offloading entry for declare target variable is incorrect: the "
4175 "address is invalid.");
4176 CGM.getDiags().Report(DiagID);
4177 continue;
4178 }
4179 // The vaiable has no definition - no need to add the entry.
4180 if (CE->getVarSize().isZero())
4181 continue;
4182 break;
4183 }
4184 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4185 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4187, __PRETTY_FUNCTION__))
4186 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4187, __PRETTY_FUNCTION__))
4187 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4187, __PRETTY_FUNCTION__))
;
4188 if (CGM.getLangOpts().OpenMPIsDevice)
4189 continue;
4190 if (!CE->getAddress()) {
4191 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4192 DiagnosticsEngine::Error,
4193 "Offloading entry for declare target variable is incorrect: the "
4194 "address is invalid.");
4195 CGM.getDiags().Report(DiagID);
4196 continue;
4197 }
4198 break;
4199 }
4200 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4201 CE->getVarSize().getQuantity(), Flags,
4202 CE->getLinkage());
4203 } else {
4204 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4204)
;
4205 }
4206 }
4207}
4208
4209/// Loads all the offload entries information from the host IR
4210/// metadata.
4211void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4212 // If we are in target mode, load the metadata from the host IR. This code has
4213 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4214
4215 if (!CGM.getLangOpts().OpenMPIsDevice)
4216 return;
4217
4218 if (CGM.getLangOpts().OMPHostIRFile.empty())
4219 return;
4220
4221 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4222 if (auto EC = Buf.getError()) {
4223 CGM.getDiags().Report(diag::err_cannot_open_file)
4224 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4225 return;
4226 }
4227
4228 llvm::LLVMContext C;
4229 auto ME = expectedToErrorOrAndEmitErrors(
4230 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4231
4232 if (auto EC = ME.getError()) {
4233 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4234 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4235 CGM.getDiags().Report(DiagID)
4236 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4237 return;
4238 }
4239
4240 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4241 if (!MD)
4242 return;
4243
4244 for (llvm::MDNode *MN : MD->operands()) {
4245 auto &&GetMDInt = [MN](unsigned Idx) {
4246 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4247 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4248 };
4249
4250 auto &&GetMDString = [MN](unsigned Idx) {
4251 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4252 return V->getString();
4253 };
4254
4255 switch (GetMDInt(0)) {
4256 default:
4257 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4257)
;
4258 break;
4259 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4260 OffloadingEntryInfoTargetRegion:
4261 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4262 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4263 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4264 /*Order=*/GetMDInt(5));
4265 break;
4266 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4267 OffloadingEntryInfoDeviceGlobalVar:
4268 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4269 /*MangledName=*/GetMDString(1),
4270 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4271 /*Flags=*/GetMDInt(2)),
4272 /*Order=*/GetMDInt(3));
4273 break;
4274 }
4275 }
4276}
4277
4278void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4279 if (!KmpRoutineEntryPtrTy) {
4280 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4281 ASTContext &C = CGM.getContext();
4282 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4283 FunctionProtoType::ExtProtoInfo EPI;
4284 KmpRoutineEntryPtrQTy = C.getPointerType(
4285 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4286 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4287 }
4288}
4289
4290QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4291 // Make sure the type of the entry is already created. This is the type we
4292 // have to create:
4293 // struct __tgt_offload_entry{
4294 // void *addr; // Pointer to the offload entry info.
4295 // // (function or global)
4296 // char *name; // Name of the function or global.
4297 // size_t size; // Size of the entry info (0 if it a function).
4298 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4299 // int32_t reserved; // Reserved, to use by the runtime library.
4300 // };
4301 if (TgtOffloadEntryQTy.isNull()) {
4302 ASTContext &C = CGM.getContext();
4303 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4304 RD->startDefinition();
4305 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4306 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4307 addFieldToRecordDecl(C, RD, C.getSizeType());
4308 addFieldToRecordDecl(
4309 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4310 addFieldToRecordDecl(
4311 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4312 RD->completeDefinition();
4313 RD->addAttr(PackedAttr::CreateImplicit(C));
4314 TgtOffloadEntryQTy = C.getRecordType(RD);
4315 }
4316 return TgtOffloadEntryQTy;
4317}
4318
4319QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4320 // These are the types we need to build:
4321 // struct __tgt_device_image{
4322 // void *ImageStart; // Pointer to the target code start.
4323 // void *ImageEnd; // Pointer to the target code end.
4324 // // We also add the host entries to the device image, as it may be useful
4325 // // for the target runtime to have access to that information.
4326 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
4327 // // the entries.
4328 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4329 // // entries (non inclusive).
4330 // };
4331 if (TgtDeviceImageQTy.isNull()) {
4332 ASTContext &C = CGM.getContext();
4333 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4334 RD->startDefinition();
4335 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4336 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4337 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4338 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4339 RD->completeDefinition();
4340 TgtDeviceImageQTy = C.getRecordType(RD);
4341 }
4342 return TgtDeviceImageQTy;
4343}
4344
4345QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4346 // struct __tgt_bin_desc{
4347 // int32_t NumDevices; // Number of devices supported.
4348 // __tgt_device_image *DeviceImages; // Arrays of device images
4349 // // (one per device).
4350 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
4351 // // entries.
4352 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4353 // // entries (non inclusive).
4354 // };
4355 if (TgtBinaryDescriptorQTy.isNull()) {
4356 ASTContext &C = CGM.getContext();
4357 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4358 RD->startDefinition();
4359 addFieldToRecordDecl(
4360 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4361 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4362 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4363 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4364 RD->completeDefinition();
4365 TgtBinaryDescriptorQTy = C.getRecordType(RD);
4366 }
4367 return TgtBinaryDescriptorQTy;
4368}
4369
4370namespace {
4371struct PrivateHelpersTy {
4372 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4373 const VarDecl *PrivateElemInit)
4374 : Original(Original), PrivateCopy(PrivateCopy),
4375 PrivateElemInit(PrivateElemInit) {}
4376 const VarDecl *Original;
4377 const VarDecl *PrivateCopy;
4378 const VarDecl *PrivateElemInit;
4379};
4380typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4381} // anonymous namespace
4382
4383static RecordDecl *
4384createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4385 if (!Privates.empty()) {
4386 ASTContext &C = CGM.getContext();
4387 // Build struct .kmp_privates_t. {
4388 // /* private vars */
4389 // };
4390 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4391 RD->startDefinition();
4392 for (const auto &Pair : Privates) {
4393 const VarDecl *VD = Pair.second.Original;
4394 QualType Type = VD->getType().getNonReferenceType();
4395 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4396 if (VD->hasAttrs()) {
4397 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4398 E(VD->getAttrs().end());
4399 I != E; ++I)
4400 FD->addAttr(*I);
4401 }
4402 }
4403 RD->completeDefinition();
4404 return RD;
4405 }
4406 return nullptr;
4407}
4408
4409static RecordDecl *
4410createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4411 QualType KmpInt32Ty,
4412 QualType KmpRoutineEntryPointerQTy) {
4413 ASTContext &C = CGM.getContext();
4414 // Build struct kmp_task_t {
4415 // void * shareds;
4416 // kmp_routine_entry_t routine;
4417 // kmp_int32 part_id;
4418 // kmp_cmplrdata_t data1;
4419 // kmp_cmplrdata_t data2;
4420 // For taskloops additional fields:
4421 // kmp_uint64 lb;
4422 // kmp_uint64 ub;
4423 // kmp_int64 st;
4424 // kmp_int32 liter;
4425 // void * reductions;
4426 // };
4427 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4428 UD->startDefinition();
4429 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4430 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4431 UD->completeDefinition();
4432 QualType KmpCmplrdataTy = C.getRecordType(UD);
4433 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4434 RD->startDefinition();
4435 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4436 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4437 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4438 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4439 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4440 if (isOpenMPTaskLoopDirective(Kind)) {
4441 QualType KmpUInt64Ty =
4442 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4443 QualType KmpInt64Ty =
4444 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4445 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4446 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4447 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4448 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4449 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4450 }
4451 RD->completeDefinition();
4452 return RD;
4453}
4454
4455static RecordDecl *
4456createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4457 ArrayRef<PrivateDataTy> Privates) {
4458 ASTContext &C = CGM.getContext();
4459 // Build struct kmp_task_t_with_privates {
4460 // kmp_task_t task_data;
4461 // .kmp_privates_t. privates;
4462 // };
4463 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4464 RD->startDefinition();
4465 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4466 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4467 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4468 RD->completeDefinition();
4469 return RD;
4470}
4471
4472/// Emit a proxy function which accepts kmp_task_t as the second
4473/// argument.
4474/// \code
4475/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4476/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4477/// For taskloops:
4478/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4479/// tt->reductions, tt->shareds);
4480/// return 0;
4481/// }
4482/// \endcode
4483static llvm::Function *
4484emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4485 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4486 QualType KmpTaskTWithPrivatesPtrQTy,
4487 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4488 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4489 llvm::Value *TaskPrivatesMap) {
4490 ASTContext &C = CGM.getContext();
4491 FunctionArgList Args;
4492 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4493 ImplicitParamDecl::Other);
4494 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4495 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4496 ImplicitParamDecl::Other);
4497 Args.push_back(&GtidArg);
4498 Args.push_back(&TaskTypeArg);
4499 const auto &TaskEntryFnInfo =
4500 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4501 llvm::FunctionType *TaskEntryTy =
4502 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4503 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4504 auto *TaskEntry = llvm::Function::Create(
4505 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4506 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4507 TaskEntry->setDoesNotRecurse();
4508 CodeGenFunction CGF(CGM);
4509 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4510 Loc, Loc);
4511
4512 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4513 // tt,
4514 // For taskloops:
4515 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4516 // tt->task_data.shareds);
4517 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4518 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4519 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4520 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4521 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4522 const auto *KmpTaskTWithPrivatesQTyRD =
4523 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4524 LValue Base =
4525 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4526 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4527 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4528 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4529 llvm::Value *PartidParam = PartIdLVal.getPointer();
4530
4531 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4532 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4533 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4534 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4535 CGF.ConvertTypeForMem(SharedsPtrTy));
4536
4537 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4538 llvm::Value *PrivatesParam;
4539 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4540 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4541 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4542 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4543 } else {
4544 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4545 }
4546
4547 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4548 TaskPrivatesMap,
4549 CGF.Builder
4550 .CreatePointerBitCastOrAddrSpaceCast(
4551 TDBase.getAddress(), CGF.VoidPtrTy)
4552 .getPointer()};
4553 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4554 std::end(CommonArgs));
4555 if (isOpenMPTaskLoopDirective(Kind)) {
4556 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4557 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4558 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4559 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4560 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4561 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4562 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4563 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4564 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4565 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4566 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4567 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4568 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4569 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4570 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4571 CallArgs.push_back(LBParam);
4572 CallArgs.push_back(UBParam);
4573 CallArgs.push_back(StParam);
4574 CallArgs.push_back(LIParam);
4575 CallArgs.push_back(RParam);
4576 }
4577 CallArgs.push_back(SharedsParam);
4578
4579 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4580 CallArgs);
4581 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4582 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4583 CGF.FinishFunction();
4584 return TaskEntry;
4585}
4586
4587static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4588 SourceLocation Loc,
4589 QualType KmpInt32Ty,
4590 QualType KmpTaskTWithPrivatesPtrQTy,
4591 QualType KmpTaskTWithPrivatesQTy) {
4592 ASTContext &C = CGM.getContext();
4593 FunctionArgList Args;
4594 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4595 ImplicitParamDecl::Other);
4596 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4597 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4598 ImplicitParamDecl::Other);
4599 Args.push_back(&GtidArg);
4600 Args.push_back(&TaskTypeArg);
4601 const auto &DestructorFnInfo =
4602 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4603 llvm::FunctionType *DestructorFnTy =
4604 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4605 std::string Name =
4606 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4607 auto *DestructorFn =
4608 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4609 Name, &CGM.getModule());
4610 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4611 DestructorFnInfo);
4612 DestructorFn->setDoesNotRecurse();
4613 CodeGenFunction CGF(CGM);
4614 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4615 Args, Loc, Loc);
4616
4617 LValue Base = CGF.EmitLoadOfPointerLValue(
4618 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4619 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4620 const auto *KmpTaskTWithPrivatesQTyRD =
4621 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4622 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4623 Base = CGF.EmitLValueForField(Base, *FI);
4624 for (const auto *Field :
4625 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4626 if (QualType::DestructionKind DtorKind =
4627 Field->getType().isDestructedType()) {
4628 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4629 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4630 }
4631 }
4632 CGF.FinishFunction();
4633 return DestructorFn;
4634}
4635
4636/// Emit a privates mapping function for correct handling of private and
4637/// firstprivate variables.
4638/// \code
4639/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4640/// **noalias priv1,..., <tyn> **noalias privn) {
4641/// *priv1 = &.privates.priv1;
4642/// ...;
4643/// *privn = &.privates.privn;
4644/// }
4645/// \endcode
4646static llvm::Value *
4647emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4648 ArrayRef<const Expr *> PrivateVars,
4649 ArrayRef<const Expr *> FirstprivateVars,
4650 ArrayRef<const Expr *> LastprivateVars,
4651 QualType PrivatesQTy,
4652 ArrayRef<PrivateDataTy> Privates) {
4653 ASTContext &C = CGM.getContext();
4654 FunctionArgList Args;
4655 ImplicitParamDecl TaskPrivatesArg(
4656 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4657 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4658 ImplicitParamDecl::Other);
4659 Args.push_back(&TaskPrivatesArg);
4660 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4661 unsigned Counter = 1;
4662 for (const Expr *E : PrivateVars) {
4663 Args.push_back(ImplicitParamDecl::Create(
4664 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4665 C.getPointerType(C.getPointerType(E->getType()))
4666 .withConst()
4667 .withRestrict(),
4668 ImplicitParamDecl::Other));
4669 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4670 PrivateVarsPos[VD] = Counter;
4671 ++Counter;
4672 }
4673 for (const Expr *E : FirstprivateVars) {
4674 Args.push_back(ImplicitParamDecl::Create(
4675 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4676 C.getPointerType(C.getPointerType(E->getType()))
4677 .withConst()
4678 .withRestrict(),
4679 ImplicitParamDecl::Other));
4680 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4681 PrivateVarsPos[VD] = Counter;
4682 ++Counter;
4683 }
4684 for (const Expr *E : LastprivateVars) {
4685 Args.push_back(ImplicitParamDecl::Create(
4686 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4687 C.getPointerType(C.getPointerType(E->getType()))
4688 .withConst()
4689 .withRestrict(),
4690 ImplicitParamDecl::Other));
4691 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4692 PrivateVarsPos[VD] = Counter;
4693 ++Counter;
4694 }
4695 const auto &TaskPrivatesMapFnInfo =
4696 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4697 llvm::FunctionType *TaskPrivatesMapTy =
4698 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4699 std::string Name =
4700 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4701 auto *TaskPrivatesMap = llvm::Function::Create(
4702 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4703 &CGM.getModule());
4704 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4705 TaskPrivatesMapFnInfo);
4706 if (CGM.getLangOpts().Optimize) {
4707 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4708 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4709 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4710 }
4711 CodeGenFunction CGF(CGM);
4712 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4713 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4714
4715 // *privi = &.privates.privi;
4716 LValue Base = CGF.EmitLoadOfPointerLValue(
4717 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4718 TaskPrivatesArg.getType()->castAs<PointerType>());
4719 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4720 Counter = 0;
4721 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4722 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4723 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4724 LValue RefLVal =
4725 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4726 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4727 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4728 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4729 ++Counter;
4730 }
4731 CGF.FinishFunction();
4732 return TaskPrivatesMap;
4733}
4734
4735/// Emit initialization for private variables in task-based directives.
4736static void emitPrivatesInit(CodeGenFunction &CGF,
4737 const OMPExecutableDirective &D,
4738 Address KmpTaskSharedsPtr, LValue TDBase,
4739 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4740 QualType SharedsTy, QualType SharedsPtrTy,
4741 const OMPTaskDataTy &Data,
4742 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4743 ASTContext &C = CGF.getContext();
4744 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4745 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4746 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4747 ? OMPD_taskloop
4748 : OMPD_task;
4749 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4750 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4751 LValue SrcBase;
4752 bool IsTargetTask =
4753 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4754 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4755 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4756 // PointersArray and SizesArray. The original variables for these arrays are
4757 // not captured and we get their addresses explicitly.
4758 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4759 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4760 SrcBase = CGF.MakeAddrLValue(
4761 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4762 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4763 SharedsTy);
4764 }
4765 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4766 for (const PrivateDataTy &Pair : Privates) {
4767 const VarDecl *VD = Pair.second.PrivateCopy;
4768 const Expr *Init = VD->getAnyInitializer();
4769 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4770 !CGF.isTrivialInitializer(Init)))) {
4771 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4772 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4773 const VarDecl *OriginalVD = Pair.second.Original;
4774 // Check if the variable is the target-based BasePointersArray,
4775 // PointersArray or SizesArray.
4776 LValue SharedRefLValue;
4777 QualType Type = PrivateLValue.getType();
4778 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4779 if (IsTargetTask && !SharedField) {
4780 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4781 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4782 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4783 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4784 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4785 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4786 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
4787 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4787, __PRETTY_FUNCTION__))
;
4788 SharedRefLValue =
4789 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4790 } else {
4791 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4792 SharedRefLValue = CGF.MakeAddrLValue(
4793 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4794 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4795 SharedRefLValue.getTBAAInfo());
4796 }
4797 if (Type->isArrayType()) {
4798 // Initialize firstprivate array.
4799 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4800 // Perform simple memcpy.
4801 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4802 } else {
4803 // Initialize firstprivate array using element-by-element
4804 // initialization.
4805 CGF.EmitOMPAggregateAssign(
4806 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4807 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4808 Address SrcElement) {
4809 // Clean up any temporaries needed by the initialization.
4810 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4811 InitScope.addPrivate(
4812 Elem, [SrcElement]() -> Address { return SrcElement; });
4813 (void)InitScope.Privatize();
4814 // Emit initialization for single element.
4815 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4816 CGF, &CapturesInfo);
4817 CGF.EmitAnyExprToMem(Init, DestElement,
4818 Init->getType().getQualifiers(),
4819 /*IsInitializer=*/false);
4820 });
4821 }
4822 } else {
4823 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4824 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4825 return SharedRefLValue.getAddress();
4826 });
4827 (void)InitScope.Privatize();
4828 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4829 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4830 /*capturedByInit=*/false);
4831 }
4832 } else {
4833 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4834 }
4835 }
4836 ++FI;
4837 }
4838}
4839
4840/// Check if duplication function is required for taskloops.
4841static bool checkInitIsRequired(CodeGenFunction &CGF,
4842 ArrayRef<PrivateDataTy> Privates) {
4843 bool InitRequired = false;
4844 for (const PrivateDataTy &Pair : Privates) {
4845 const VarDecl *VD = Pair.second.PrivateCopy;
4846 const Expr *Init = VD->getAnyInitializer();
4847 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4848 !CGF.isTrivialInitializer(Init));
4849 if (InitRequired)
4850 break;
4851 }
4852 return InitRequired;
4853}
4854
4855
4856/// Emit task_dup function (for initialization of
4857/// private/firstprivate/lastprivate vars and last_iter flag)
4858/// \code
4859/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4860/// lastpriv) {
4861/// // setup lastprivate flag
4862/// task_dst->last = lastpriv;
4863/// // could be constructor calls here...
4864/// }
4865/// \endcode
4866static llvm::Value *
4867emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4868 const OMPExecutableDirective &D,
4869 QualType KmpTaskTWithPrivatesPtrQTy,
4870 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4871 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4872 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4873 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4874 ASTContext &C = CGM.getContext();
4875 FunctionArgList Args;
4876 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4877 KmpTaskTWithPrivatesPtrQTy,
4878 ImplicitParamDecl::Other);
4879 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4880 KmpTaskTWithPrivatesPtrQTy,
4881 ImplicitParamDecl::Other);
4882 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4883 ImplicitParamDecl::Other);
4884 Args.push_back(&DstArg);
4885 Args.push_back(&SrcArg);
4886 Args.push_back(&LastprivArg);
4887 const auto &TaskDupFnInfo =
4888 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4889 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4890 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4891 auto *TaskDup = llvm::Function::Create(
4892 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4893 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4894 TaskDup->setDoesNotRecurse();
4895 CodeGenFunction CGF(CGM);
4896 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4897 Loc);
4898
4899 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4900 CGF.GetAddrOfLocalVar(&DstArg),
4901 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4902 // task_dst->liter = lastpriv;
4903 if (WithLastIter) {
4904 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4905 LValue Base = CGF.EmitLValueForField(
4906 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4907 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4908 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4909 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4910 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4911 }
4912
4913 // Emit initial values for private copies (if any).
4914 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4914, __PRETTY_FUNCTION__))
;
4915 Address KmpTaskSharedsPtr = Address::invalid();
4916 if (!Data.FirstprivateVars.empty()) {
4917 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4918 CGF.GetAddrOfLocalVar(&SrcArg),
4919 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4920 LValue Base = CGF.EmitLValueForField(
4921 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4922 KmpTaskSharedsPtr = Address(
4923 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4924 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4925 KmpTaskTShareds)),
4926 Loc),
4927 CGF.getNaturalTypeAlignment(SharedsTy));
4928 }
4929 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4930 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4931 CGF.FinishFunction();
4932 return TaskDup;
4933}
4934
4935/// Checks if destructor function is required to be generated.
4936/// \return true if cleanups are required, false otherwise.
4937static bool
4938checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4939 bool NeedsCleanup = false;
4940 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4941 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4942 for (const FieldDecl *FD : PrivateRD->fields()) {
4943 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4944 if (NeedsCleanup)
4945 break;
4946 }
4947 return NeedsCleanup;
4948}
4949
4950CGOpenMPRuntime::TaskResultTy
4951CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4952 const OMPExecutableDirective &D,
4953 llvm::Function *TaskFunction, QualType SharedsTy,
4954 Address Shareds, const OMPTaskDataTy &Data) {
4955 ASTContext &C = CGM.getContext();
4956 llvm::SmallVector<PrivateDataTy, 4> Privates;
4957 // Aggregate privates and sort them by the alignment.
4958 auto I = Data.PrivateCopies.begin();
4959 for (const Expr *E : Data.PrivateVars) {
4960 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4961 Privates.emplace_back(
4962 C.getDeclAlign(VD),
4963 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4964 /*PrivateElemInit=*/nullptr));
4965 ++I;
4966 }
4967 I = Data.FirstprivateCopies.begin();
4968 auto IElemInitRef = Data.FirstprivateInits.begin();
4969 for (const Expr *E : Data.FirstprivateVars) {
4970 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4971 Privates.emplace_back(
4972 C.getDeclAlign(VD),
4973 PrivateHelpersTy(
4974 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4975 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4976 ++I;
4977 ++IElemInitRef;
4978 }
4979 I = Data.LastprivateCopies.begin();
4980 for (const Expr *E : Data.LastprivateVars) {
4981 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4982 Privates.emplace_back(
4983 C.getDeclAlign(VD),
4984 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4985 /*PrivateElemInit=*/nullptr));
4986 ++I;
4987 }
4988 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
4989 return L.first > R.first;
4990 });
4991 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4992 // Build type kmp_routine_entry_t (if not built yet).
4993 emitKmpRoutineEntryT(KmpInt32Ty);
4994 // Build type kmp_task_t (if not built yet).
4995 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4996 if (SavedKmpTaskloopTQTy.isNull()) {
4997 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4998 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4999 }
5000 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5001 } else {
5002 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5005, __PRETTY_FUNCTION__))
5003 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5005, __PRETTY_FUNCTION__))
5004 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5005, __PRETTY_FUNCTION__))
5005 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5005, __PRETTY_FUNCTION__))
;
5006 if (SavedKmpTaskTQTy.isNull()) {
5007 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5008 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5009 }
5010 KmpTaskTQTy = SavedKmpTaskTQTy;
5011 }
5012 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5013 // Build particular struct kmp_task_t for the given task.
5014 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5015 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5016 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5017 QualType KmpTaskTWithPrivatesPtrQTy =
5018 C.getPointerType(KmpTaskTWithPrivatesQTy);
5019 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5020 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5021 KmpTaskTWithPrivatesTy->getPointerTo();
5022 llvm::Value *KmpTaskTWithPrivatesTySize =
5023 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5024 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5025
5026 // Emit initial values for private copies (if any).
5027 llvm::Value *TaskPrivatesMap = nullptr;
5028 llvm::Type *TaskPrivatesMapTy =
5029 std::next(TaskFunction->arg_begin(), 3)->getType();
5030 if (!Privates.empty()) {
5031 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5032 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5033 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5034 FI->getType(), Privates);
5035 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5036 TaskPrivatesMap, TaskPrivatesMapTy);
5037 } else {
5038 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5039 cast<llvm::PointerType>(TaskPrivatesMapTy));
5040 }
5041 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5042 // kmp_task_t *tt);
5043 llvm::Function *TaskEntry = emitProxyTaskFunction(
5044 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5045 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5046 TaskPrivatesMap);
5047
5048 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5049 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5050 // kmp_routine_entry_t *task_entry);
5051 // Task flags. Format is taken from
5052 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5053 // description of kmp_tasking_flags struct.
5054 enum {
5055 TiedFlag = 0x1,
5056 FinalFlag = 0x2,
5057 DestructorsFlag = 0x8,
5058 PriorityFlag = 0x20
5059 };
5060 unsigned Flags = Data.Tied ? TiedFlag : 0;
5061 bool NeedsCleanup = false;
5062 if (!Privates.empty()) {
5063 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5064 if (NeedsCleanup)
5065 Flags = Flags | DestructorsFlag;
5066 }
5067 if (Data.Priority.getInt())
5068 Flags = Flags | PriorityFlag;
5069 llvm::Value *TaskFlags =
5070 Data.Final.getPointer()
5071 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5072 CGF.Builder.getInt32(FinalFlag),
5073 CGF.Builder.getInt32(/*C=*/0))
5074 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5075 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5076 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5077 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
5078 getThreadID(CGF, Loc), TaskFlags,
5079 KmpTaskTWithPrivatesTySize, SharedsSize,
5080 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5081 TaskEntry, KmpRoutineEntryPtrTy)};
5082 llvm::Value *NewTask = CGF.EmitRuntimeCall(
5083 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5084 llvm::Value *NewTaskNewTaskTTy =
5085 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5086 NewTask, KmpTaskTWithPrivatesPtrTy);
5087 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5088 KmpTaskTWithPrivatesQTy);
5089 LValue TDBase =
5090 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5091 // Fill the data in the resulting kmp_task_t record.
5092 // Copy shareds if there are any.
5093 Address KmpTaskSharedsPtr = Address::invalid();
5094 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5095 KmpTaskSharedsPtr =
5096 Address(CGF.EmitLoadOfScalar(
5097 CGF.EmitLValueForField(
5098 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5099 KmpTaskTShareds)),
5100 Loc),
5101 CGF.getNaturalTypeAlignment(SharedsTy));
5102 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5103 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5104 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5105 }
5106 // Emit initial values for private copies (if any).
5107 TaskResultTy Result;
5108 if (!Privates.empty()) {
5109 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5110 SharedsTy, SharedsPtrTy, Data, Privates,
5111 /*ForDup=*/false);
5112 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5113 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5114 Result.TaskDupFn = emitTaskDupFunction(
5115 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5116 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5117 /*WithLastIter=*/!Data.LastprivateVars.empty());
5118 }
5119 }
5120 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5121 enum { Priority = 0, Destructors = 1 };
5122 // Provide pointer to function with destructors for privates.
5123 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5124 const RecordDecl *KmpCmplrdataUD =
5125 (*FI)->getType()->getAsUnionType()->getDecl();
5126 if (NeedsCleanup) {
5127 llvm::Value *DestructorFn = emitDestructorsFunction(
5128 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5129 KmpTaskTWithPrivatesQTy);
5130 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5131 LValue DestructorsLV = CGF.EmitLValueForField(
5132 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5133 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5134 DestructorFn, KmpRoutineEntryPtrTy),
5135 DestructorsLV);
5136 }
5137 // Set priority.
5138 if (Data.Priority.getInt()) {
5139 LValue Data2LV = CGF.EmitLValueForField(
5140 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5141 LValue PriorityLV = CGF.EmitLValueForField(
5142 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5143 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5144 }
5145 Result.NewTask = NewTask;
5146 Result.TaskEntry = TaskEntry;
5147 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5148 Result.TDBase = TDBase;
5149 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5150 return Result;
5151}
5152
5153void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5154 const OMPExecutableDirective &D,
5155 llvm::Function *TaskFunction,
5156 QualType SharedsTy, Address Shareds,
5157 const Expr *IfCond,
5158 const OMPTaskDataTy &Data) {
5159 if (!CGF.HaveInsertPoint())
5160 return;
5161
5162 TaskResultTy Result =
5163 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5164 llvm::Value *NewTask = Result.NewTask;
5165 llvm::Function *TaskEntry = Result.TaskEntry;
5166 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5167 LValue TDBase = Result.TDBase;
5168 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5169 ASTContext &C = CGM.getContext();
5170 // Process list of dependences.
5171 Address DependenciesArray = Address::invalid();
5172 unsigned NumDependencies = Data.Dependences.size();
5173 if (NumDependencies) {
5174 // Dependence kind for RTL.
5175 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5176 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5177 RecordDecl *KmpDependInfoRD;
5178 QualType FlagsTy =
5179 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5180 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5181 if (KmpDependInfoTy.isNull()) {
5182 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5183 KmpDependInfoRD->startDefinition();
5184 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5185 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5186 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5187 KmpDependInfoRD->completeDefinition();
5188 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5189 } else {
5190 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5191 }
5192 // Define type kmp_depend_info[<Dependences.size()>];
5193 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5194 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5195 ArrayType::Normal, /*IndexTypeQuals=*/0);
5196 // kmp_depend_info[<Dependences.size()>] deps;
5197 DependenciesArray =
5198 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5199 for (unsigned I = 0; I < NumDependencies; ++I) {
5200 const Expr *E = Data.Dependences[I].second;
5201 LValue Addr = CGF.EmitLValue(E);
5202 llvm::Value *Size;
5203 QualType Ty = E->getType();
5204 if (const auto *ASE =
5205 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5206 LValue UpAddrLVal =
5207 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
5208 llvm::Value *UpAddr =
5209 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5210 llvm::Value *LowIntPtr =
5211 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5212 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5213 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5214 } else {
5215 Size = CGF.getTypeSize(Ty);
5216 }
5217 LValue Base = CGF.MakeAddrLValue(
5218 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5219 KmpDependInfoTy);
5220 // deps[i].base_addr = &<Dependences[i].second>;
5221 LValue BaseAddrLVal = CGF.EmitLValueForField(
5222 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5223 CGF.EmitStoreOfScalar(
5224 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5225 BaseAddrLVal);
5226 // deps[i].len = sizeof(<Dependences[i].second>);
5227 LValue LenLVal = CGF.EmitLValueForField(
5228 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5229 CGF.EmitStoreOfScalar(Size, LenLVal);
5230 // deps[i].flags = <Dependences[i].first>;
5231 RTLDependenceKindTy DepKind;
5232 switch (Data.Dependences[I].first) {
5233 case OMPC_DEPEND_in:
5234 DepKind = DepIn;
5235 break;
5236 // Out and InOut dependencies must use the same code.
5237 case OMPC_DEPEND_out:
5238 case OMPC_DEPEND_inout:
5239 DepKind = DepInOut;
5240 break;
5241 case OMPC_DEPEND_mutexinoutset:
5242 DepKind = DepMutexInOutSet;
5243 break;
5244 case OMPC_DEPEND_source:
5245 case OMPC_DEPEND_sink:
5246 case OMPC_DEPEND_unknown:
5247 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-9~svn362543/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5247)
;
5248 }
5249 LValue FlagsLVal = CGF.EmitLValueForField(
5250 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5251 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5252 FlagsLVal);
5253 }
5254 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5255 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5256 }
5257
5258 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5259 // libcall.
5260 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5261 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5262 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5263 // list is not empty
5264 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5265 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5266 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5267 llvm::Value *DepTaskArgs[7];
5268 if (NumDependencies) {
5269 DepTaskArgs[0] = UpLoc;
5270 DepTaskArgs[1] = ThreadID;
5271 DepTaskArgs[2] = NewTask;
5272 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5273 DepTaskArgs[4] = DependenciesArray.getPointer();
5274 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5275 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5276 }
5277 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5278 &TaskArgs,
5279 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5280 if (!Data.Tied) {
5281 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5282 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5283 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5284 }
5285 if (NumDependencies) {
5286 CGF.EmitRuntimeCall(
5287 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5288 } else {
5289 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5290 TaskArgs);
5291 }
5292 // Check if parent region is untied and build return for untied task;
5293 if (auto *Region =
5294 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5295 Region->emitUntiedSwitch(CGF);
5296 };
5297
5298 llvm::Value *DepWaitTaskArgs[6];
5299 if (NumDependencies) {
5300 DepWaitTaskArgs[0] = UpLoc;
5301 DepWaitTaskArgs[1] = ThreadID;
5302 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5303 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5304 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5305 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5306 }
5307 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5308 NumDependencies, &DepWaitTaskArgs,
5309 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5310 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5311 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5312 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5313 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5314 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5315 // is specified.
5316 if (NumDependencies)
5317 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5318 DepWaitTaskArgs);
5319 // Call proxy_task_entry(gtid, new_task);
5320 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5321 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5322 Action.Enter(CGF);
5323 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5324 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5325 OutlinedFnArgs);
5326 };
5327
5328 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5329 // kmp_task_t *new_task);
5330 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5331 // kmp_task_t *new_task);
5332 RegionCodeGenTy RCG(CodeGen);
5333 CommonActionTy Action(
5334 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5335 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5336 RCG.setAction(Action);
5337 RCG(CGF);
5338 };
5339
5340 if (IfCond) {
5341 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5342 } else {
5343 RegionCodeGenTy ThenRCG(ThenCodeGen);
5344 ThenRCG(CGF);
5345 }
5346}
5347
5348void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5349 const OMPLoopDirective &D,
5350 llvm::Function *TaskFunction,
5351 QualType SharedsTy, Address Shareds,
5352 const Expr *IfCond,
5353 const OMPTaskDataTy &Data) {
5354 if (!CGF.HaveInsertPoint())
5355 return;
5356 TaskResultTy Result =
5357 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5358 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5359 // libcall.
5360 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5361 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5362 // sched, kmp_uint64 grainsize, void *task_dup);
5363 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5364 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5365 llvm::Value *IfVal;
5366 if (IfCond) {
5367 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5368 /*isSigned=*/true);
5369 } else {
5370 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5371 }
5372
5373 LValue LBLVal = CGF.EmitLValueForField(
5374 Result.TDBase,
5375 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5376 const auto *LBVar =
5377 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5378 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5379 /*IsInitializer=*/true);
5380 LValue UBLVal = CGF.EmitLValueForField(
5381 Result.TDBase,
5382 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5383 const auto *UBVar =
5384 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5385 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5386 /*IsInitializer=*/true);
5387 LValue StLVal = CGF.EmitLValueForField(
5388 Result.TDBase,
5389 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5390 const auto *StVar =
5391 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5392 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5393 /*IsInitializer=*/true);
5394 // Store reductions address.
5395 LValue RedLVal = CGF.EmitLValueForField(
5396 Result.TDBase,
5397 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5398 if (Data.Reductions) {
5399 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5400 } else {
5401 CGF.EmitNullInitialization(RedLVal.getAddress(),
5402 CGF.getContext().VoidPtrTy);
5403 }
5404 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5405 llvm::Value *TaskArgs[] = {
5406 UpLoc,
5407 ThreadID,
5408 Result.NewTask,
5409 IfVal,
5410 LBLVal.getPointer(),
5411 UBLVal.getPointer(),
5412 CGF.EmitLoadOfScalar(StLVal, Loc),
5413 llvm::ConstantInt::getSigned(
5414 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5415 llvm::ConstantInt::getSigned(
5416 CGF.IntTy, Data.Schedule.getPointer()
5417 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5418 : NoSchedule),
5419 Data.Schedule.getPointer()
5420 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5421 /*isSigned=*/false)
5422 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5423 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5424 Result.TaskDupFn, CGF.VoidPtrTy)
5425 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5426 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5427}
5428
5429/// Emit reduction operation for each element of array (required for
5430/// array sections) LHS op = RHS.
5431/// \param Type Type of array.
5432/// \param LHSVar Variable on the left side of the reduction operation
5433/// (references element of array in original variable).
5434/// \param RHSVar Variable on the right side of the reduction operation
5435/// (references element of array in original variable).
5436/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5437/// RHSVar.
5438static void EmitOMPAggregateReduction(
5439 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5440 const VarDecl *RHSVar,
5441 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5442 const Expr *, const Expr *)> &RedOpGen,
5443 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5444 const Expr *UpExpr = nullptr) {
5445 // Perform element-by-element initialization.
5446 QualType ElementTy;
5447 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5448 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5449
5450 // Drill down to the base element type on both arrays.
5451 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5452 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5453
5454 llvm::Value *RHSBegin = RHSAddr.getPointer();
5455 llvm::Value *LHSBegin = LHSAddr.getPointer();
5456 // Cast from pointer to array type to pointer to single element.
5457 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5458 // The basic structure here is a while-do loop.
5459 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5460 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5461 llvm::Value *IsEmpty =
5462 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5463 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5464
5465 // Enter the loop body, making that address the current address.
5466 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5467 CGF.EmitBlock(BodyBB);
5468
5469 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5470
5471 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5472 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5473 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5474 Address RHSElementCurrent =
5475 Address(RHSElementPHI,
5476 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5477
5478 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5479 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5480 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5481 Address LHSElementCurrent =
5482 Address(LHSElementPHI,
5483 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5484
5485 // Emit copy.
5486 CodeGenFunction::OMPPrivateScope Scope(CGF);
5487 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5488 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5489 Scope.Privatize();
5490 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5491 Scope.ForceCleanup();
5492
5493 // Shift the address forward by one element.
5494 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5495 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5496 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5497 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5498 // Check whether we've reached the end.
5499 llvm::Value *Done =
5500 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5501 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5502 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5503 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5504
5505 // Done.
5506 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5507}
5508
5509/// Emit reduction combiner. If the combiner is a simple expression emit it as
5510/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5511/// UDR combiner function.
5512static void emitReductionCombiner(CodeGenFunction &CGF,
5513 const Expr *ReductionOp) {
5514 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5515 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5516 if (const auto *DRE =
5517 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5518 if (const auto *DRD =
5519 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5520 std::pair<llvm::Function *, llvm::Function *> Reduction =
5521 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5522 RValue Func = RValue::get(Reduction.first);
5523 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5524 CGF.EmitIgnoredExpr(ReductionOp);
5525 return;
5526 }
5527 CGF.EmitIgnoredExpr(ReductionOp);
5528}
5529
5530llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5531 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5532 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5533 ArrayRef<const Expr *> ReductionOps) {
5534 ASTContext &C = CGM.getContext();
5535
5536 // void reduction_func(void *LHSArg, void *RHSArg);
5537 FunctionArgList Args;
5538 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5539 ImplicitParamDecl::Other);
5540 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5541 ImplicitParamDecl::Other);
5542 Args.push_back(&LHSArg);
5543 Args.push_back(&RHSArg);
5544 const auto &CGFI =
5545 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5546 std::string Name = getName({"omp", "reduction", "reduction_func"});
5547 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5548 llvm::GlobalValue::InternalLinkage, Name,
5549 &CGM.getModule());
5550 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5551 Fn->setDoesNotRecurse();
5552 CodeGenFunction CGF(CGM);
5553 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5554
5555 // Dst = (void*[n])(LHSArg);
5556 // Src = (void*[n])(RHSArg);
5557 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5558 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5559 ArgsType), CGF.getPointerAlign());
5560 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5561 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5562 ArgsType), CGF.getPointerAlign());
5563
5564 // ...
5565 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5566 // ...
5567 CodeGenFunction::OMPPrivateScope Scope(CGF);
5568 auto IPriv = Privates.begin();
5569 unsigned Idx = 0;
5570 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5571 const auto *RHSVar =
5572 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5573 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5574 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5575 });
5576 const auto *LHSVar =
5577 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5578 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5579 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5580 });
5581 QualType PrivTy = (*IPriv)->getType();
5582 if (PrivTy->isVariablyModifiedType()) {
5583 // Get array size and emit VLA type.
5584 ++Idx;
5585 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5586 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5587 const VariableArrayType *VLA =
5588 CGF.getContext().getAsVariableArrayType(PrivTy);
5589 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5590 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5591 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5592 CGF.EmitVariablyModifiedType(PrivTy);
5593 }
5594 }
5595 Scope.Privatize();
5596 IPriv = Privates.begin();
5597 auto ILHS = LHSExprs.begin();
5598 auto IRHS = RHSExprs.begin();
5599 for (const Expr *E : ReductionOps) {
5600 if ((*IPriv)->getType()->isArrayType()) {
5601 // Emit reduction for array section.
5602 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5603 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5604 EmitOMPAggregateReduction(
5605 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5606 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5607 emitReductionCombiner(CGF, E);
5608 });
5609 } else {
5610 // Emit reduction for array subscript or single variable.
5611 emitReductionCombiner(CGF, E);
5612 }
5613 ++IPriv;
5614 ++ILHS;
5615 ++IRHS;
5616 }
5617 Scope.ForceCleanup();
5618 CGF.FinishFunction();
5619 return Fn;
5620}
5621
5622void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5623 const Expr *ReductionOp,
5624 const Expr *PrivateRef,
5625 const DeclRefExpr *LHS,
5626 const DeclRefExpr *RHS) {
5627 if (PrivateRef->getType()->isArrayType()) {
5628 // Emit reduction for array section.
5629 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5630 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5631 EmitOMPAggregateReduction(
5632 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5633 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5634 emitReductionCombiner(CGF, ReductionOp);
5635 });
5636 } else {
5637 // Emit reduction for array subscript or single variable.
5638 emitReductionCombiner(CGF, ReductionOp);
5639 }
5640}
5641
5642void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5643 ArrayRef<const Expr *> Privates,
5644 ArrayRef<const Expr *> LHSExprs,
5645 ArrayRef<const Expr *> RHSExprs,
5646 ArrayRef<const Expr *> ReductionOps,
5647 ReductionOptionsTy Options) {
5648 if (!CGF.HaveInsertPoint())
5649 return;
5650
5651 bool WithNowait = Options.WithNowait;
5652 bool SimpleReduction = Options.SimpleReduction;
5653
5654 // Next code should be emitted for reduction:
5655 //
5656 // static kmp_critical_name lock = { 0 };
5657 //
5658 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5659 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5660 // ...
5661 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5662 // *(Type<n>-1*)rhs[<n>-1]);
5663 // }
5664 //
5665 // ...
5666 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5667 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5668 // RedList, reduce_func, &<lock>)) {
5669 // case 1:
5670 // ...
5671 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5672 // ...
5673 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5674 // break;
5675 // case 2:
5676 // ...
5677 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5678 // ...
5679 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5680 // break;
5681 // default:;
5682 // }
5683 //
5684 // if SimpleReduction is true, only the next code is generated:
5685 // ...
5686 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5687 // ...
5688
5689 ASTContext &C = CGM.getContext();
5690
5691 if (SimpleReduction) {
5692 CodeGenFunction::RunCleanupsScope Scope(CGF);
5693 auto IPriv = Privates.begin();
5694 auto ILHS = LHSExprs.begin();
5695 auto IRHS = RHSExprs.begin();
5696 for (const Expr *E : ReductionOps) {
5697 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5698 cast<DeclRefExpr>(*IRHS));
5699 ++IPriv;
5700 ++ILHS;
5701 ++IRHS;
5702 }
5703 return;
5704 }
5705
5706 // 1. Build a list of reduction variables.
5707 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5708 auto Size = RHSExprs.size();
5709 for (const Expr *E : Privates) {
5710 if (E->getType()->isVariablyModifiedType())
5711 // Reserve place for array size.
5712 ++Size;
5713 }
5714 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5715 QualType ReductionArrayTy =
5716 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
5717 /*IndexTypeQuals=*/0);
5718 Address ReductionList =
5719 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5720 auto IPriv = Privates.begin();
5721 unsigned Idx = 0;
5722 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5723 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5724 CGF.Builder.CreateStore(
5725 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5726 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5727 Elem);
5728 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5729 // Store array size.
5730 ++Idx;
5731 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5732 llvm::Value *Size = CGF.Builder.CreateIntCast(
5733 CGF.getVLASize(
5734 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5735 .NumElts,
5736 CGF.SizeTy, /*isSigned=*/false);
5737 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5738 Elem);
5739 }
5740 }
5741
5742 // 2. Emit reduce_func().
5743 llvm::Function *ReductionFn = emitReductionFunction(
5744 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5745 LHSExprs, RHSExprs, ReductionOps);
5746
5747 // 3. Create static kmp_critical_name lock = { 0 };
5748 std::string Name = getName({"reduction"});
5749 llvm::Value *Lock = getCriticalRegionLock(Name);
5750
5751 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5752 // RedList, reduce_func, &<lock>);
5753 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5754 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5755 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5756 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5757 ReductionList.getPointer(), CGF.VoidPtrTy);
5758 llvm::Value *Args[] = {
5759 IdentTLoc, // ident_t *<loc>
5760 ThreadId, // i32 <gtid>
5761 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5762 ReductionArrayTySize, // size_type sizeof(RedList)
5763 RL, // void *RedList
5764 ReductionFn, // void (*) (void *, void *) <reduce_func>