Bug Summary

File:tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 2883, column 20
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn373517/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn373517=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-02-234743-9763-1 -x c++ /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGOpenMPRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/CodeGen/ConstantInitBuilder.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/StmtOpenMP.h"
21#include "clang/Basic/BitmaskEnum.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30
31using namespace clang;
32using namespace CodeGen;
33
34namespace {
35/// Base class for handling code generation inside OpenMP regions.
36class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
37public:
38 /// Kinds of OpenMP regions used in codegen.
39 enum CGOpenMPRegionKind {
40 /// Region with outlined function for standalone 'parallel'
41 /// directive.
42 ParallelOutlinedRegion,
43 /// Region with outlined function for standalone 'task' directive.
44 TaskOutlinedRegion,
45 /// Region for constructs that do not require function outlining,
46 /// like 'for', 'sections', 'atomic' etc. directives.
47 InlinedRegion,
48 /// Region with outlined function for standalone 'target' directive.
49 TargetRegion,
50 };
51
52 CGOpenMPRegionInfo(const CapturedStmt &CS,
53 const CGOpenMPRegionKind RegionKind,
54 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
55 bool HasCancel)
56 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58
59 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
60 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61 bool HasCancel)
62 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
64
65 /// Get a variable or parameter for storing global thread id
66 /// inside OpenMP construct.
67 virtual const VarDecl *getThreadIDVariable() const = 0;
68
69 /// Emit the captured statement body.
70 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
71
72 /// Get an LValue for the current ThreadID variable.
73 /// \return LValue for thread id variable. This LValue always has type int32*.
74 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75
76 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
77
78 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
79
80 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
81
82 bool hasCancel() const { return HasCancel; }
83
84 static bool classof(const CGCapturedStmtInfo *Info) {
85 return Info->getKind() == CR_OpenMP;
86 }
87
88 ~CGOpenMPRegionInfo() override = default;
89
90protected:
91 CGOpenMPRegionKind RegionKind;
92 RegionCodeGenTy CodeGen;
93 OpenMPDirectiveKind Kind;
94 bool HasCancel;
95};
96
97/// API for captured statement code generation in OpenMP constructs.
98class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
99public:
100 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
101 const RegionCodeGenTy &CodeGen,
102 OpenMPDirectiveKind Kind, bool HasCancel,
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
105 HasCancel),
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 107, __PRETTY_FUNCTION__))
;
108 }
109
110 /// Get a variable or parameter for storing global thread id
111 /// inside OpenMP construct.
112 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
113
114 /// Get the name of the capture helper.
115 StringRef getHelperName() const override { return HelperName; }
116
117 static bool classof(const CGCapturedStmtInfo *Info) {
118 return CGOpenMPRegionInfo::classof(Info) &&
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
121 }
122
123private:
124 /// A variable or parameter storing global thread id for OpenMP
125 /// constructs.
126 const VarDecl *ThreadIDVar;
127 StringRef HelperName;
128};
129
130/// API for captured statement code generation in OpenMP constructs.
131class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
132public:
133 class UntiedTaskActionTy final : public PrePostActionTy {
134 bool Untied;
135 const VarDecl *PartIDVar;
136 const RegionCodeGenTy UntiedCodeGen;
137 llvm::SwitchInst *UntiedSwitch = nullptr;
138
139 public:
140 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
141 const RegionCodeGenTy &UntiedCodeGen)
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
143 void Enter(CodeGenFunction &CGF) override {
144 if (Untied) {
145 // Emit task switching point.
146 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
147 CGF.GetAddrOfLocalVar(PartIDVar),
148 PartIDVar->getType()->castAs<PointerType>());
149 llvm::Value *Res =
150 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
151 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
152 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153 CGF.EmitBlock(DoneBB);
154 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157 CGF.Builder.GetInsertBlock());
158 emitUntiedSwitch(CGF);
159 }
160 }
161 void emitUntiedSwitch(CodeGenFunction &CGF) const {
162 if (Untied) {
163 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
164 CGF.GetAddrOfLocalVar(PartIDVar),
165 PartIDVar->getType()->castAs<PointerType>());
166 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167 PartIdLVal);
168 UntiedCodeGen(CGF);
169 CodeGenFunction::JumpDest CurPoint =
170 CGF.getJumpDestInCurrentScope(".untied.next.");
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174 CGF.Builder.GetInsertBlock());
175 CGF.EmitBranchThroughCleanup(CurPoint);
176 CGF.EmitBlock(CurPoint.getBlock());
177 }
178 }
179 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180 };
181 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
182 const VarDecl *ThreadIDVar,
183 const RegionCodeGenTy &CodeGen,
184 OpenMPDirectiveKind Kind, bool HasCancel,
185 const UntiedTaskActionTy &Action)
186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187 ThreadIDVar(ThreadIDVar), Action(Action) {
188 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 188, __PRETTY_FUNCTION__))
;
189 }
190
191 /// Get a variable or parameter for storing global thread id
192 /// inside OpenMP construct.
193 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
194
195 /// Get an LValue for the current ThreadID variable.
196 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197
198 /// Get the name of the capture helper.
199 StringRef getHelperName() const override { return ".omp_outlined."; }
200
201 void emitUntiedSwitch(CodeGenFunction &CGF) override {
202 Action.emitUntiedSwitch(CGF);
203 }
204
205 static bool classof(const CGCapturedStmtInfo *Info) {
206 return CGOpenMPRegionInfo::classof(Info) &&
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208 TaskOutlinedRegion;
209 }
210
211private:
212 /// A variable or parameter storing global thread id for OpenMP
213 /// constructs.
214 const VarDecl *ThreadIDVar;
215 /// Action for emitting code for untied tasks.
216 const UntiedTaskActionTy &Action;
217};
218
219/// API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
223 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
224 const RegionCodeGenTy &CodeGen,
225 OpenMPDirectiveKind Kind, bool HasCancel)
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OldCSI(OldCSI),
228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229
230 // Retrieve the value of the context parameter.
231 llvm::Value *getContextValue() const override {
232 if (OuterRegionInfo)
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 234)
;
235 }
236
237 void setContextValue(llvm::Value *V) override {
238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
240 return;
241 }
242 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 242)
;
243 }
244
245 /// Lookup the captured field decl for a variable.
246 const FieldDecl *lookup(const VarDecl *VD) const override {
247 if (OuterRegionInfo)
248 return OuterRegionInfo->lookup(VD);
249 // If there is no outer outlined region,no need to lookup in a list of
250 // captured variables, we can use the original one.
251 return nullptr;
252 }
253
254 FieldDecl *getThisFieldDecl() const override {
255 if (OuterRegionInfo)
256 return OuterRegionInfo->getThisFieldDecl();
257 return nullptr;
258 }
259
260 /// Get a variable or parameter for storing global thread id
261 /// inside OpenMP construct.
262 const VarDecl *getThreadIDVariable() const override {
263 if (OuterRegionInfo)
264 return OuterRegionInfo->getThreadIDVariable();
265 return nullptr;
266 }
267
268 /// Get an LValue for the current ThreadID variable.
269 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270 if (OuterRegionInfo)
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 272)
;
273 }
274
275 /// Get the name of the capture helper.
276 StringRef getHelperName() const override {
277 if (auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
279 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 279)
;
280 }
281
282 void emitUntiedSwitch(CodeGenFunction &CGF) override {
283 if (OuterRegionInfo)
284 OuterRegionInfo->emitUntiedSwitch(CGF);
285 }
286
287 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
289 static bool classof(const CGCapturedStmtInfo *Info) {
290 return CGOpenMPRegionInfo::classof(Info) &&
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292 }
293
294 ~CGOpenMPInlinedRegionInfo() override = default;
295
296private:
297 /// CodeGen info about outer OpenMP region.
298 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299 CGOpenMPRegionInfo *OuterRegionInfo;
300};
301
302/// API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
308public:
309 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
310 const RegionCodeGenTy &CodeGen, StringRef HelperName)
311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
312 /*HasCancel=*/false),
313 HelperName(HelperName) {}
314
315 /// This is unused for target regions because each starts executing
316 /// with a single thread.
317 const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319 /// Get the name of the capture helper.
320 StringRef getHelperName() const override { return HelperName; }
321
322 static bool classof(const CGCapturedStmtInfo *Info) {
323 return CGOpenMPRegionInfo::classof(Info) &&
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325 }
326
327private:
328 StringRef HelperName;
329};
330
331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
332 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 332)
;
333}
334/// API for generation of expressions captured in a innermost OpenMP
335/// region.
336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
337public:
338 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340 OMPD_unknown,
341 /*HasCancel=*/false),
342 PrivScope(CGF) {
343 // Make sure the globals captured in the provided statement are local by
344 // using the privatization logic. We assume the same variable is not
345 // captured more than once.
346 for (const auto &C : CS.captures()) {
347 if (!C.capturesVariable() && !C.capturesVariableByCopy())
348 continue;
349
350 const VarDecl *VD = C.getCapturedVar();
351 if (VD->isLocalVarDeclOrParm())
352 continue;
353
354 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
355 /*RefersToEnclosingVariableOrCapture=*/false,
356 VD->getType().getNonReferenceType(), VK_LValue,
357 C.getLocation());
358 PrivScope.addPrivate(
359 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
360 }
361 (void)PrivScope.Privatize();
362 }
363
364 /// Lookup the captured field decl for a variable.
365 const FieldDecl *lookup(const VarDecl *VD) const override {
366 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
367 return FD;
368 return nullptr;
369 }
370
371 /// Emit the captured statement body.
372 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
373 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 373)
;
374 }
375
376 /// Get a variable or parameter for storing global thread id
377 /// inside OpenMP construct.
378 const VarDecl *getThreadIDVariable() const override {
379 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 379)
;
380 }
381
382 /// Get the name of the capture helper.
383 StringRef getHelperName() const override {
384 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 384)
;
385 }
386
387 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
388
389private:
390 /// Private scope to capture global variables.
391 CodeGenFunction::OMPPrivateScope PrivScope;
392};
393
394/// RAII for emitting code of OpenMP constructs.
395class InlinedOpenMPRegionRAII {
396 CodeGenFunction &CGF;
397 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398 FieldDecl *LambdaThisCaptureField = nullptr;
399 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
400
401public:
402 /// Constructs region for combined constructs.
403 /// \param CodeGen Code generation sequence for combined directives. Includes
404 /// a list of functions used for code generation of implicitly inlined
405 /// regions.
406 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
407 OpenMPDirectiveKind Kind, bool HasCancel)
408 : CGF(CGF) {
409 // Start emission for the construct.
410 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414 CGF.LambdaThisCaptureField = nullptr;
415 BlockInfo = CGF.BlockInfo;
416 CGF.BlockInfo = nullptr;
417 }
418
419 ~InlinedOpenMPRegionRAII() {
420 // Restore original CapturedStmtInfo only if we're done with code emission.
421 auto *OldCSI =
422 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
423 delete CGF.CapturedStmtInfo;
424 CGF.CapturedStmtInfo = OldCSI;
425 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
426 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
427 CGF.BlockInfo = BlockInfo;
428 }
429};
430
431/// Values for bit flags used in the ident_t to describe the fields.
432/// All enumeric elements are named and described in accordance with the code
433/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
434enum OpenMPLocationFlags : unsigned {
435 /// Use trampoline for internal microtask.
436 OMP_IDENT_IMD = 0x01,
437 /// Use c-style ident structure.
438 OMP_IDENT_KMPC = 0x02,
439 /// Atomic reduction option for kmpc_reduce.
440 OMP_ATOMIC_REDUCE = 0x10,
441 /// Explicit 'barrier' directive.
442 OMP_IDENT_BARRIER_EXPL = 0x20,
443 /// Implicit barrier in code.
444 OMP_IDENT_BARRIER_IMPL = 0x40,
445 /// Implicit barrier in 'for' directive.
446 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
447 /// Implicit barrier in 'sections' directive.
448 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
449 /// Implicit barrier in 'single' directive.
450 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
451 /// Call of __kmp_for_static_init for static loop.
452 OMP_IDENT_WORK_LOOP = 0x200,
453 /// Call of __kmp_for_static_init for sections.
454 OMP_IDENT_WORK_SECTIONS = 0x400,
455 /// Call of __kmp_for_static_init for distribute.
456 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
458};
459
460namespace {
461LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
462/// Values for bit flags for marking which requires clauses have been used.
463enum OpenMPOffloadingRequiresDirFlags : int64_t {
464 /// flag undefined.
465 OMP_REQ_UNDEFINED = 0x000,
466 /// no requires clause present.
467 OMP_REQ_NONE = 0x001,
468 /// reverse_offload clause.
469 OMP_REQ_REVERSE_OFFLOAD = 0x002,
470 /// unified_address clause.
471 OMP_REQ_UNIFIED_ADDRESS = 0x004,
472 /// unified_shared_memory clause.
473 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
474 /// dynamic_allocators clause.
475 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
476 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
477};
478
479enum OpenMPOffloadingReservedDeviceIDs {
480 /// Device ID if the device was not defined, runtime should get it
481 /// from environment variables in the spec.
482 OMP_DEVICEID_UNDEF = -1,
483};
484} // anonymous namespace
485
486/// Describes ident structure that describes a source location.
487/// All descriptions are taken from
488/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
489/// Original structure:
490/// typedef struct ident {
491/// kmp_int32 reserved_1; /**< might be used in Fortran;
492/// see above */
493/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
494/// KMP_IDENT_KMPC identifies this union
495/// member */
496/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
497/// see above */
498///#if USE_ITT_BUILD
499/// /* but currently used for storing
500/// region-specific ITT */
501/// /* contextual information. */
502///#endif /* USE_ITT_BUILD */
503/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
504/// C++ */
505/// char const *psource; /**< String describing the source location.
506/// The string is composed of semi-colon separated
507// fields which describe the source file,
508/// the function and a pair of line numbers that
509/// delimit the construct.
510/// */
511/// } ident_t;
512enum IdentFieldIndex {
513 /// might be used in Fortran
514 IdentField_Reserved_1,
515 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
516 IdentField_Flags,
517 /// Not really used in Fortran any more
518 IdentField_Reserved_2,
519 /// Source[4] in Fortran, do not use for C++
520 IdentField_Reserved_3,
521 /// String describing the source location. The string is composed of
522 /// semi-colon separated fields which describe the source file, the function
523 /// and a pair of line numbers that delimit the construct.
524 IdentField_PSource
525};
526
527/// Schedule types for 'omp for' loops (these enumerators are taken from
528/// the enum sched_type in kmp.h).
529enum OpenMPSchedType {
530 /// Lower bound for default (unordered) versions.
531 OMP_sch_lower = 32,
532 OMP_sch_static_chunked = 33,
533 OMP_sch_static = 34,
534 OMP_sch_dynamic_chunked = 35,
535 OMP_sch_guided_chunked = 36,
536 OMP_sch_runtime = 37,
537 OMP_sch_auto = 38,
538 /// static with chunk adjustment (e.g., simd)
539 OMP_sch_static_balanced_chunked = 45,
540 /// Lower bound for 'ordered' versions.
541 OMP_ord_lower = 64,
542 OMP_ord_static_chunked = 65,
543 OMP_ord_static = 66,
544 OMP_ord_dynamic_chunked = 67,
545 OMP_ord_guided_chunked = 68,
546 OMP_ord_runtime = 69,
547 OMP_ord_auto = 70,
548 OMP_sch_default = OMP_sch_static,
549 /// dist_schedule types
550 OMP_dist_sch_static_chunked = 91,
551 OMP_dist_sch_static = 92,
552 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
553 /// Set if the monotonic schedule modifier was present.
554 OMP_sch_modifier_monotonic = (1 << 29),
555 /// Set if the nonmonotonic schedule modifier was present.
556 OMP_sch_modifier_nonmonotonic = (1 << 30),
557};
558
559enum OpenMPRTLFunction {
560 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
561 /// kmpc_micro microtask, ...);
562 OMPRTL__kmpc_fork_call,
563 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
564 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
565 OMPRTL__kmpc_threadprivate_cached,
566 /// Call to void __kmpc_threadprivate_register( ident_t *,
567 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
568 OMPRTL__kmpc_threadprivate_register,
569 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
570 OMPRTL__kmpc_global_thread_num,
571 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
572 // kmp_critical_name *crit);
573 OMPRTL__kmpc_critical,
574 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
575 // global_tid, kmp_critical_name *crit, uintptr_t hint);
576 OMPRTL__kmpc_critical_with_hint,
577 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
578 // kmp_critical_name *crit);
579 OMPRTL__kmpc_end_critical,
580 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
581 // global_tid);
582 OMPRTL__kmpc_cancel_barrier,
583 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
584 OMPRTL__kmpc_barrier,
585 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
586 OMPRTL__kmpc_for_static_fini,
587 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
588 // global_tid);
589 OMPRTL__kmpc_serialized_parallel,
590 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
591 // global_tid);
592 OMPRTL__kmpc_end_serialized_parallel,
593 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
594 // kmp_int32 num_threads);
595 OMPRTL__kmpc_push_num_threads,
596 // Call to void __kmpc_flush(ident_t *loc);
597 OMPRTL__kmpc_flush,
598 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
599 OMPRTL__kmpc_master,
600 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
601 OMPRTL__kmpc_end_master,
602 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
603 // int end_part);
604 OMPRTL__kmpc_omp_taskyield,
605 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
606 OMPRTL__kmpc_single,
607 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
608 OMPRTL__kmpc_end_single,
609 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
610 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
611 // kmp_routine_entry_t *task_entry);
612 OMPRTL__kmpc_omp_task_alloc,
613 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
614 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
615 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
616 // kmp_int64 device_id);
617 OMPRTL__kmpc_omp_target_task_alloc,
618 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
619 // new_task);
620 OMPRTL__kmpc_omp_task,
621 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
622 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
623 // kmp_int32 didit);
624 OMPRTL__kmpc_copyprivate,
625 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
626 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
627 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
628 OMPRTL__kmpc_reduce,
629 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
630 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
631 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
632 // *lck);
633 OMPRTL__kmpc_reduce_nowait,
634 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
635 // kmp_critical_name *lck);
636 OMPRTL__kmpc_end_reduce,
637 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
638 // kmp_critical_name *lck);
639 OMPRTL__kmpc_end_reduce_nowait,
640 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
641 // kmp_task_t * new_task);
642 OMPRTL__kmpc_omp_task_begin_if0,
643 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
644 // kmp_task_t * new_task);
645 OMPRTL__kmpc_omp_task_complete_if0,
646 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
647 OMPRTL__kmpc_ordered,
648 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
649 OMPRTL__kmpc_end_ordered,
650 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
651 // global_tid);
652 OMPRTL__kmpc_omp_taskwait,
653 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
654 OMPRTL__kmpc_taskgroup,
655 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
656 OMPRTL__kmpc_end_taskgroup,
657 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
658 // int proc_bind);
659 OMPRTL__kmpc_push_proc_bind,
660 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
661 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
662 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
663 OMPRTL__kmpc_omp_task_with_deps,
664 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
665 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
666 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
667 OMPRTL__kmpc_omp_wait_deps,
668 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
669 // global_tid, kmp_int32 cncl_kind);
670 OMPRTL__kmpc_cancellationpoint,
671 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
672 // kmp_int32 cncl_kind);
673 OMPRTL__kmpc_cancel,
674 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
675 // kmp_int32 num_teams, kmp_int32 thread_limit);
676 OMPRTL__kmpc_push_num_teams,
677 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
678 // microtask, ...);
679 OMPRTL__kmpc_fork_teams,
680 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
681 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
682 // sched, kmp_uint64 grainsize, void *task_dup);
683 OMPRTL__kmpc_taskloop,
684 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
685 // num_dims, struct kmp_dim *dims);
686 OMPRTL__kmpc_doacross_init,
687 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
688 OMPRTL__kmpc_doacross_fini,
689 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
690 // *vec);
691 OMPRTL__kmpc_doacross_post,
692 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
693 // *vec);
694 OMPRTL__kmpc_doacross_wait,
695 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
696 // *data);
697 OMPRTL__kmpc_task_reduction_init,
698 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
699 // *d);
700 OMPRTL__kmpc_task_reduction_get_th_data,
701 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
702 OMPRTL__kmpc_alloc,
703 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
704 OMPRTL__kmpc_free,
705
706 //
707 // Offloading related calls
708 //
709 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
710 // size);
711 OMPRTL__kmpc_push_target_tripcount,
712 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
713 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
714 // *arg_types);
715 OMPRTL__tgt_target,
716 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
717 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
718 // *arg_types);
719 OMPRTL__tgt_target_nowait,
720 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
721 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
722 // *arg_types, int32_t num_teams, int32_t thread_limit);
723 OMPRTL__tgt_target_teams,
724 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
725 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
726 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
727 OMPRTL__tgt_target_teams_nowait,
728 // Call to void __tgt_register_requires(int64_t flags);
729 OMPRTL__tgt_register_requires,
730 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
731 OMPRTL__tgt_register_lib,
732 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
733 OMPRTL__tgt_unregister_lib,
734 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
735 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
736 OMPRTL__tgt_target_data_begin,
737 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
738 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
739 // *arg_types);
740 OMPRTL__tgt_target_data_begin_nowait,
741 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
742 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
743 OMPRTL__tgt_target_data_end,
744 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
745 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
746 // *arg_types);
747 OMPRTL__tgt_target_data_end_nowait,
748 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
749 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
750 OMPRTL__tgt_target_data_update,
751 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
752 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
753 // *arg_types);
754 OMPRTL__tgt_target_data_update_nowait,
755 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
756 OMPRTL__tgt_mapper_num_components,
757 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
758 // *base, void *begin, int64_t size, int64_t type);
759 OMPRTL__tgt_push_mapper_component,
760};
761
762/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
763/// region.
764class CleanupTy final : public EHScopeStack::Cleanup {
765 PrePostActionTy *Action;
766
767public:
768 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
769 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
770 if (!CGF.HaveInsertPoint())
771 return;
772 Action->Exit(CGF);
773 }
774};
775
776} // anonymous namespace
777
778void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
779 CodeGenFunction::RunCleanupsScope Scope(CGF);
780 if (PrePostAction) {
781 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
782 Callback(CodeGen, CGF, *PrePostAction);
783 } else {
784 PrePostActionTy Action;
785 Callback(CodeGen, CGF, Action);
786 }
787}
788
789/// Check if the combiner is a call to UDR combiner and if it is so return the
790/// UDR decl used for reduction.
791static const OMPDeclareReductionDecl *
792getReductionInit(const Expr *ReductionOp) {
793 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
794 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
795 if (const auto *DRE =
796 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
797 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
798 return DRD;
799 return nullptr;
800}
801
802static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
803 const OMPDeclareReductionDecl *DRD,
804 const Expr *InitOp,
805 Address Private, Address Original,
806 QualType Ty) {
807 if (DRD->getInitializer()) {
808 std::pair<llvm::Function *, llvm::Function *> Reduction =
809 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
810 const auto *CE = cast<CallExpr>(InitOp);
811 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
812 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
813 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
814 const auto *LHSDRE =
815 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
816 const auto *RHSDRE =
817 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
818 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
819 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
820 [=]() { return Private; });
821 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
822 [=]() { return Original; });
823 (void)PrivateScope.Privatize();
824 RValue Func = RValue::get(Reduction.second);
825 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
826 CGF.EmitIgnoredExpr(InitOp);
827 } else {
828 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
829 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
830 auto *GV = new llvm::GlobalVariable(
831 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
832 llvm::GlobalValue::PrivateLinkage, Init, Name);
833 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
834 RValue InitRVal;
835 switch (CGF.getEvaluationKind(Ty)) {
836 case TEK_Scalar:
837 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
838 break;
839 case TEK_Complex:
840 InitRVal =
841 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
842 break;
843 case TEK_Aggregate:
844 InitRVal = RValue::getAggregate(LV.getAddress());
845 break;
846 }
847 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
848 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
849 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
850 /*IsInitializer=*/false);
851 }
852}
853
854/// Emit initialization of arrays of complex types.
855/// \param DestAddr Address of the array.
856/// \param Type Type of array.
857/// \param Init Initial expression of array.
858/// \param SrcAddr Address of the original array.
859static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
860 QualType Type, bool EmitDeclareReductionInit,
861 const Expr *Init,
862 const OMPDeclareReductionDecl *DRD,
863 Address SrcAddr = Address::invalid()) {
864 // Perform element-by-element initialization.
865 QualType ElementTy;
866
867 // Drill down to the base element type on both arrays.
868 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
869 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
870 DestAddr =
871 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
872 if (DRD)
873 SrcAddr =
874 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
875
876 llvm::Value *SrcBegin = nullptr;
877 if (DRD)
878 SrcBegin = SrcAddr.getPointer();
879 llvm::Value *DestBegin = DestAddr.getPointer();
880 // Cast from pointer to array type to pointer to single element.
881 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
882 // The basic structure here is a while-do loop.
883 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
884 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
885 llvm::Value *IsEmpty =
886 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
887 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
888
889 // Enter the loop body, making that address the current address.
890 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
891 CGF.EmitBlock(BodyBB);
892
893 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
894
895 llvm::PHINode *SrcElementPHI = nullptr;
896 Address SrcElementCurrent = Address::invalid();
897 if (DRD) {
898 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
899 "omp.arraycpy.srcElementPast");
900 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
901 SrcElementCurrent =
902 Address(SrcElementPHI,
903 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
904 }
905 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
906 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
907 DestElementPHI->addIncoming(DestBegin, EntryBB);
908 Address DestElementCurrent =
909 Address(DestElementPHI,
910 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
911
912 // Emit copy.
913 {
914 CodeGenFunction::RunCleanupsScope InitScope(CGF);
915 if (EmitDeclareReductionInit) {
916 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
917 SrcElementCurrent, ElementTy);
918 } else
919 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
920 /*IsInitializer=*/false);
921 }
922
923 if (DRD) {
924 // Shift the address forward by one element.
925 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
926 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
927 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
928 }
929
930 // Shift the address forward by one element.
931 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
932 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
933 // Check whether we've reached the end.
934 llvm::Value *Done =
935 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
936 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
937 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
938
939 // Done.
940 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
941}
942
943LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
944 return CGF.EmitOMPSharedLValue(E);
945}
946
947LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
948 const Expr *E) {
949 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
950 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
951 return LValue();
952}
953
954void ReductionCodeGen::emitAggregateInitialization(
955 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
956 const OMPDeclareReductionDecl *DRD) {
957 // Emit VarDecl with copy init for arrays.
958 // Get the address of the original variable captured in current
959 // captured region.
960 const auto *PrivateVD =
961 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
962 bool EmitDeclareReductionInit =
963 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
964 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
965 EmitDeclareReductionInit,
966 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
967 : PrivateVD->getInit(),
968 DRD, SharedLVal.getAddress());
969}
970
971ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
972 ArrayRef<const Expr *> Privates,
973 ArrayRef<const Expr *> ReductionOps) {
974 ClausesData.reserve(Shareds.size());
975 SharedAddresses.reserve(Shareds.size());
976 Sizes.reserve(Shareds.size());
977 BaseDecls.reserve(Shareds.size());
978 auto IPriv = Privates.begin();
979 auto IRed = ReductionOps.begin();
980 for (const Expr *Ref : Shareds) {
981 ClausesData.emplace_back(Ref, *IPriv, *IRed);
982 std::advance(IPriv, 1);
983 std::advance(IRed, 1);
984 }
985}
986
987void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
988 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
989 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
;
990 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
991 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
992 SharedAddresses.emplace_back(First, Second);
993}
994
995void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
996 const auto *PrivateVD =
997 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
998 QualType PrivateType = PrivateVD->getType();
999 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1000 if (!PrivateType->isVariablyModifiedType()) {
1001 Sizes.emplace_back(
1002 CGF.getTypeSize(
1003 SharedAddresses[N].first.getType().getNonReferenceType()),
1004 nullptr);
1005 return;
1006 }
1007 llvm::Value *Size;
1008 llvm::Value *SizeInChars;
1009 auto *ElemType =
1010 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
1011 ->getElementType();
1012 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1013 if (AsArraySection) {
1014 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
1015 SharedAddresses[N].first.getPointer());
1016 Size = CGF.Builder.CreateNUWAdd(
1017 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1018 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1019 } else {
1020 SizeInChars = CGF.getTypeSize(
1021 SharedAddresses[N].first.getType().getNonReferenceType());
1022 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1023 }
1024 Sizes.emplace_back(SizeInChars, Size);
1025 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1026 CGF,
1027 cast<OpaqueValueExpr>(
1028 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1029 RValue::get(Size));
1030 CGF.EmitVariablyModifiedType(PrivateType);
1031}
1032
1033void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1034 llvm::Value *Size) {
1035 const auto *PrivateVD =
1036 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1037 QualType PrivateType = PrivateVD->getType();
1038 if (!PrivateType->isVariablyModifiedType()) {
1039 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1040 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1041 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
;
1042 return;
1043 }
1044 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1045 CGF,
1046 cast<OpaqueValueExpr>(
1047 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1048 RValue::get(Size));
1049 CGF.EmitVariablyModifiedType(PrivateType);
1050}
1051
1052void ReductionCodeGen::emitInitialization(
1053 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1054 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1055 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1055, __PRETTY_FUNCTION__))
;
1056 const auto *PrivateVD =
1057 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1058 const OMPDeclareReductionDecl *DRD =
1059 getReductionInit(ClausesData[N].ReductionOp);
1060 QualType PrivateType = PrivateVD->getType();
1061 PrivateAddr = CGF.Builder.CreateElementBitCast(
1062 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1063 QualType SharedType = SharedAddresses[N].first.getType();
1064 SharedLVal = CGF.MakeAddrLValue(
1065 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1066 CGF.ConvertTypeForMem(SharedType)),
1067 SharedType, SharedAddresses[N].first.getBaseInfo(),
1068 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1069 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1070 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1071 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1072 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1073 PrivateAddr, SharedLVal.getAddress(),
1074 SharedLVal.getType());
1075 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1076 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1077 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1078 PrivateVD->getType().getQualifiers(),
1079 /*IsInitializer=*/false);
1080 }
1081}
1082
1083bool ReductionCodeGen::needCleanups(unsigned N) {
1084 const auto *PrivateVD =
1085 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1086 QualType PrivateType = PrivateVD->getType();
1087 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1088 return DTorKind != QualType::DK_none;
1089}
1090
1091void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1092 Address PrivateAddr) {
1093 const auto *PrivateVD =
1094 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1095 QualType PrivateType = PrivateVD->getType();
1096 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1097 if (needCleanups(N)) {
1098 PrivateAddr = CGF.Builder.CreateElementBitCast(
1099 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1100 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1101 }
1102}
1103
1104static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1105 LValue BaseLV) {
1106 BaseTy = BaseTy.getNonReferenceType();
1107 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1108 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1109 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1110 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1111 } else {
1112 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1113 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1114 }
1115 BaseTy = BaseTy->getPointeeType();
1116 }
1117 return CGF.MakeAddrLValue(
1118 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1119 CGF.ConvertTypeForMem(ElTy)),
1120 BaseLV.getType(), BaseLV.getBaseInfo(),
1121 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1122}
1123
1124static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1125 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1126 llvm::Value *Addr) {
1127 Address Tmp = Address::invalid();
1128 Address TopTmp = Address::invalid();
1129 Address MostTopTmp = Address::invalid();
1130 BaseTy = BaseTy.getNonReferenceType();
1131 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1132 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1133 Tmp = CGF.CreateMemTemp(BaseTy);
1134 if (TopTmp.isValid())
1135 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1136 else
1137 MostTopTmp = Tmp;
1138 TopTmp = Tmp;
1139 BaseTy = BaseTy->getPointeeType();
1140 }
1141 llvm::Type *Ty = BaseLVType;
1142 if (Tmp.isValid())
1143 Ty = Tmp.getElementType();
1144 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1145 if (Tmp.isValid()) {
1146 CGF.Builder.CreateStore(Addr, Tmp);
1147 return MostTopTmp;
1148 }
1149 return Address(Addr, BaseLVAlignment);
1150}
1151
1152static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1153 const VarDecl *OrigVD = nullptr;
1154 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1155 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1156 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1157 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1158 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1159 Base = TempASE->getBase()->IgnoreParenImpCasts();
1160 DE = cast<DeclRefExpr>(Base);
1161 OrigVD = cast<VarDecl>(DE->getDecl());
1162 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1163 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1164 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1165 Base = TempASE->getBase()->IgnoreParenImpCasts();
1166 DE = cast<DeclRefExpr>(Base);
1167 OrigVD = cast<VarDecl>(DE->getDecl());
1168 }
1169 return OrigVD;
1170}
1171
1172Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1173 Address PrivateAddr) {
1174 const DeclRefExpr *DE;
1175 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1176 BaseDecls.emplace_back(OrigVD);
1177 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1178 LValue BaseLValue =
1179 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1180 OriginalBaseLValue);
1181 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1182 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1183 llvm::Value *PrivatePointer =
1184 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1185 PrivateAddr.getPointer(),
1186 SharedAddresses[N].first.getAddress().getType());
1187 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1188 return castToBase(CGF, OrigVD->getType(),
1189 SharedAddresses[N].first.getType(),
1190 OriginalBaseLValue.getAddress().getType(),
1191 OriginalBaseLValue.getAlignment(), Ptr);
1192 }
1193 BaseDecls.emplace_back(
1194 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1195 return PrivateAddr;
1196}
1197
1198bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1199 const OMPDeclareReductionDecl *DRD =
1200 getReductionInit(ClausesData[N].ReductionOp);
1201 return DRD && DRD->getInitializer();
1202}
1203
1204LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1205 return CGF.EmitLoadOfPointerLValue(
1206 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1207 getThreadIDVariable()->getType()->castAs<PointerType>());
1208}
1209
1210void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1211 if (!CGF.HaveInsertPoint())
1212 return;
1213 // 1.2.2 OpenMP Language Terminology
1214 // Structured block - An executable statement with a single entry at the
1215 // top and a single exit at the bottom.
1216 // The point of exit cannot be a branch out of the structured block.
1217 // longjmp() and throw() must not violate the entry/exit criteria.
1218 CGF.EHStack.pushTerminate();
1219 CodeGen(CGF);
1220 CGF.EHStack.popTerminate();
1221}
1222
1223LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1224 CodeGenFunction &CGF) {
1225 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1226 getThreadIDVariable()->getType(),
1227 AlignmentSource::Decl);
1228}
1229
1230static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1231 QualType FieldTy) {
1232 auto *Field = FieldDecl::Create(
1233 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1234 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1235 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1236 Field->setAccess(AS_public);
1237 DC->addDecl(Field);
1238 return Field;
1239}
1240
1241CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1242 StringRef Separator)
1243 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1244 OffloadEntriesInfoManager(CGM) {
1245 ASTContext &C = CGM.getContext();
1246 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1247 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1248 RD->startDefinition();
1249 // reserved_1
1250 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1251 // flags
1252 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1253 // reserved_2
1254 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1255 // reserved_3
1256 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1257 // psource
1258 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1259 RD->completeDefinition();
1260 IdentQTy = C.getRecordType(RD);
1261 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1262 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1263
1264 loadOffloadInfoMetadata();
1265}
1266
1267static bool tryEmitAlias(CodeGenModule &CGM, const GlobalDecl &NewGD,
1268 const GlobalDecl &OldGD, llvm::GlobalValue *OrigAddr,
1269 bool IsForDefinition) {
1270 // Emit at least a definition for the aliasee if the the address of the
1271 // original function is requested.
1272 if (IsForDefinition || OrigAddr)
1273 (void)CGM.GetAddrOfGlobal(NewGD);
1274 StringRef NewMangledName = CGM.getMangledName(NewGD);
1275 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1276 if (Addr && !Addr->isDeclaration()) {
1277 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1278 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD);
1279 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1280
1281 // Create a reference to the named value. This ensures that it is emitted
1282 // if a deferred decl.
1283 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1284
1285 // Create the new alias itself, but don't set a name yet.
1286 auto *GA =
1287 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1288
1289 if (OrigAddr) {
1290 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1290, __PRETTY_FUNCTION__))
;
1291
1292 GA->takeName(OrigAddr);
1293 OrigAddr->replaceAllUsesWith(
1294 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1295 OrigAddr->eraseFromParent();
1296 } else {
1297 GA->setName(CGM.getMangledName(OldGD));
1298 }
1299
1300 // Set attributes which are particular to an alias; this is a
1301 // specialization of the attributes which may be set on a global function.
1302 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1303 D->isWeakImported())
1304 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1305
1306 CGM.SetCommonAttributes(OldGD, GA);
1307 return true;
1308 }
1309 return false;
1310}
1311
1312void CGOpenMPRuntime::clear() {
1313 InternalVars.clear();
1314 // Clean non-target variable declarations possibly used only in debug info.
1315 for (const auto &Data : EmittedNonTargetVariables) {
1316 if (!Data.getValue().pointsToAliveValue())
1317 continue;
1318 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1319 if (!GV)
1320 continue;
1321 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1322 continue;
1323 GV->eraseFromParent();
1324 }
1325 // Emit aliases for the deferred aliasees.
1326 for (const auto &Pair : DeferredVariantFunction) {
1327 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1328 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1329 // If not able to emit alias, just emit original declaration.
1330 (void)tryEmitAlias(CGM, Pair.second.first, Pair.second.second, Addr,
1331 /*IsForDefinition=*/false);
1332 }
1333}
1334
1335std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1336 SmallString<128> Buffer;
1337 llvm::raw_svector_ostream OS(Buffer);
1338 StringRef Sep = FirstSeparator;
1339 for (StringRef Part : Parts) {
1340 OS << Sep << Part;
1341 Sep = Separator;
1342 }
1343 return OS.str();
1344}
1345
1346static llvm::Function *
1347emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1348 const Expr *CombinerInitializer, const VarDecl *In,
1349 const VarDecl *Out, bool IsCombiner) {
1350 // void .omp_combiner.(Ty *in, Ty *out);
1351 ASTContext &C = CGM.getContext();
1352 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1353 FunctionArgList Args;
1354 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1355 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1356 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1357 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1358 Args.push_back(&OmpOutParm);
1359 Args.push_back(&OmpInParm);
1360 const CGFunctionInfo &FnInfo =
1361 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1362 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1363 std::string Name = CGM.getOpenMPRuntime().getName(
1364 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1365 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1366 Name, &CGM.getModule());
1367 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1368 if (CGM.getLangOpts().Optimize) {
1369 Fn->removeFnAttr(llvm::Attribute::NoInline);
1370 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1371 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1372 }
1373 CodeGenFunction CGF(CGM);
1374 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1375 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1376 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1377 Out->getLocation());
1378 CodeGenFunction::OMPPrivateScope Scope(CGF);
1379 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1380 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1381 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1382 .getAddress();
1383 });
1384 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1385 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1386 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1387 .getAddress();
1388 });
1389 (void)Scope.Privatize();
1390 if (!IsCombiner && Out->hasInit() &&
1391 !CGF.isTrivialInitializer(Out->getInit())) {
1392 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1393 Out->getType().getQualifiers(),
1394 /*IsInitializer=*/true);
1395 }
1396 if (CombinerInitializer)
1397 CGF.EmitIgnoredExpr(CombinerInitializer);
1398 Scope.ForceCleanup();
1399 CGF.FinishFunction();
1400 return Fn;
1401}
1402
1403void CGOpenMPRuntime::emitUserDefinedReduction(
1404 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1405 if (UDRMap.count(D) > 0)
1406 return;
1407 llvm::Function *Combiner = emitCombinerOrInitializer(
1408 CGM, D->getType(), D->getCombiner(),
1409 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1410 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1411 /*IsCombiner=*/true);
1412 llvm::Function *Initializer = nullptr;
1413 if (const Expr *Init = D->getInitializer()) {
1414 Initializer = emitCombinerOrInitializer(
1415 CGM, D->getType(),
1416 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1417 : nullptr,
1418 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1419 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1420 /*IsCombiner=*/false);
1421 }
1422 UDRMap.try_emplace(D, Combiner, Initializer);
1423 if (CGF) {
1424 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1425 Decls.second.push_back(D);
1426 }
1427}
1428
1429std::pair<llvm::Function *, llvm::Function *>
1430CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1431 auto I = UDRMap.find(D);
1432 if (I != UDRMap.end())
1433 return I->second;
1434 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1435 return UDRMap.lookup(D);
1436}
1437
1438static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1439 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1440 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1441 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1442 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1443, __PRETTY_FUNCTION__))
1443 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1443, __PRETTY_FUNCTION__))
;
1444 CodeGenFunction CGF(CGM, true);
1445 bool HasCancel = false;
1446 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1447 HasCancel = OPD->hasCancel();
1448 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1449 HasCancel = OPSD->hasCancel();
1450 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1451 HasCancel = OPFD->hasCancel();
1452 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1453 HasCancel = OPFD->hasCancel();
1454 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1455 HasCancel = OPFD->hasCancel();
1456 else if (const auto *OPFD =
1457 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1458 HasCancel = OPFD->hasCancel();
1459 else if (const auto *OPFD =
1460 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1461 HasCancel = OPFD->hasCancel();
1462 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1463 HasCancel, OutlinedHelperName);
1464 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1465 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1466}
1467
1468llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1469 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1470 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1471 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1472 return emitParallelOrTeamsOutlinedFunction(
1473 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1474}
1475
1476llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1477 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1478 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1479 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1480 return emitParallelOrTeamsOutlinedFunction(
1481 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1482}
1483
1484llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1485 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1486 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1487 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1488 bool Tied, unsigned &NumberOfParts) {
1489 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1490 PrePostActionTy &) {
1491 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1492 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1493 llvm::Value *TaskArgs[] = {
1494 UpLoc, ThreadID,
1495 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1496 TaskTVar->getType()->castAs<PointerType>())
1497 .getPointer()};
1498 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1499 };
1500 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1501 UntiedCodeGen);
1502 CodeGen.setAction(Action);
1503 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1504, __PRETTY_FUNCTION__))
1504 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1504, __PRETTY_FUNCTION__))
;
1505 const OpenMPDirectiveKind Region =
1506 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1507 : OMPD_task;
1508 const CapturedStmt *CS = D.getCapturedStmt(Region);
1509 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1510 CodeGenFunction CGF(CGM, true);
1511 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1512 InnermostKind,
1513 TD ? TD->hasCancel() : false, Action);
1514 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1515 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1516 if (!Tied)
1517 NumberOfParts = Action.getNumberOfParts();
1518 return Res;
1519}
1520
1521static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1522 const RecordDecl *RD, const CGRecordLayout &RL,
1523 ArrayRef<llvm::Constant *> Data) {
1524 llvm::StructType *StructTy = RL.getLLVMType();
1525 unsigned PrevIdx = 0;
1526 ConstantInitBuilder CIBuilder(CGM);
1527 auto DI = Data.begin();
1528 for (const FieldDecl *FD : RD->fields()) {
1529 unsigned Idx = RL.getLLVMFieldNo(FD);
1530 // Fill the alignment.
1531 for (unsigned I = PrevIdx; I < Idx; ++I)
1532 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1533 PrevIdx = Idx + 1;
1534 Fields.add(*DI);
1535 ++DI;
1536 }
1537}
1538
1539template <class... As>
1540static llvm::GlobalVariable *
1541createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1542 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1543 As &&... Args) {
1544 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1545 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1546 ConstantInitBuilder CIBuilder(CGM);
1547 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1548 buildStructValue(Fields, CGM, RD, RL, Data);
1549 return Fields.finishAndCreateGlobal(
1550 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1551 std::forward<As>(Args)...);
1552}
1553
1554template <typename T>
1555static void
1556createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1557 ArrayRef<llvm::Constant *> Data,
1558 T &Parent) {
1559 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1560 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1561 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1562 buildStructValue(Fields, CGM, RD, RL, Data);
1563 Fields.finishAndAddTo(Parent);
1564}
1565
1566Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1567 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1568 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1569 FlagsTy FlagsKey(Flags, Reserved2Flags);
1570 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1571 if (!Entry) {
1572 if (!DefaultOpenMPPSource) {
1573 // Initialize default location for psource field of ident_t structure of
1574 // all ident_t objects. Format is ";file;function;line;column;;".
1575 // Taken from
1576 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1577 DefaultOpenMPPSource =
1578 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1579 DefaultOpenMPPSource =
1580 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1581 }
1582
1583 llvm::Constant *Data[] = {
1584 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1585 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1586 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1587 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1588 llvm::GlobalValue *DefaultOpenMPLocation =
1589 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1590 llvm::GlobalValue::PrivateLinkage);
1591 DefaultOpenMPLocation->setUnnamedAddr(
1592 llvm::GlobalValue::UnnamedAddr::Global);
1593
1594 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1595 }
1596 return Address(Entry, Align);
1597}
1598
1599void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1600 bool AtCurrentPoint) {
1601 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1602 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1602, __PRETTY_FUNCTION__))
;
1603
1604 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1605 if (AtCurrentPoint) {
1606 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1607 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1608 } else {
1609 Elem.second.ServiceInsertPt =
1610 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1611 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1612 }
1613}
1614
1615void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1616 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1617 if (Elem.second.ServiceInsertPt) {
1618 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1619 Elem.second.ServiceInsertPt = nullptr;
1620 Ptr->eraseFromParent();
1621 }
1622}
1623
1624llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1625 SourceLocation Loc,
1626 unsigned Flags) {
1627 Flags |= OMP_IDENT_KMPC;
1628 // If no debug info is generated - return global default location.
1629 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1630 Loc.isInvalid())
1631 return getOrCreateDefaultLocation(Flags).getPointer();
1632
1633 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1633, __PRETTY_FUNCTION__))
;
1634
1635 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1636 Address LocValue = Address::invalid();
1637 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1638 if (I != OpenMPLocThreadIDMap.end())
1639 LocValue = Address(I->second.DebugLoc, Align);
1640
1641 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1642 // GetOpenMPThreadID was called before this routine.
1643 if (!LocValue.isValid()) {
1644 // Generate "ident_t .kmpc_loc.addr;"
1645 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1646 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1647 Elem.second.DebugLoc = AI.getPointer();
1648 LocValue = AI;
1649
1650 if (!Elem.second.ServiceInsertPt)
1651 setLocThreadIdInsertPt(CGF);
1652 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1653 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1654 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1655 CGF.getTypeSize(IdentQTy));
1656 }
1657
1658 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1659 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1660 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1661 LValue PSource =
1662 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1663
1664 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1665 if (OMPDebugLoc == nullptr) {
1666 SmallString<128> Buffer2;
1667 llvm::raw_svector_ostream OS2(Buffer2);
1668 // Build debug location
1669 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1670 OS2 << ";" << PLoc.getFilename() << ";";
1671 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1672 OS2 << FD->getQualifiedNameAsString();
1673 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1674 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1675 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1676 }
1677 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1678 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1679
1680 // Our callers always pass this to a runtime function, so for
1681 // convenience, go ahead and return a naked pointer.
1682 return LocValue.getPointer();
1683}
1684
1685llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1686 SourceLocation Loc) {
1687 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1687, __PRETTY_FUNCTION__))
;
1688
1689 llvm::Value *ThreadID = nullptr;
1690 // Check whether we've already cached a load of the thread id in this
1691 // function.
1692 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1693 if (I != OpenMPLocThreadIDMap.end()) {
1694 ThreadID = I->second.ThreadID;
1695 if (ThreadID != nullptr)
1696 return ThreadID;
1697 }
1698 // If exceptions are enabled, do not use parameter to avoid possible crash.
1699 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1700 !CGF.getLangOpts().CXXExceptions ||
1701 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1702 if (auto *OMPRegionInfo =
1703 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1704 if (OMPRegionInfo->getThreadIDVariable()) {
1705 // Check if this an outlined function with thread id passed as argument.
1706 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1707 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1708 // If value loaded in entry block, cache it and use it everywhere in
1709 // function.
1710 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1711 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1712 Elem.second.ThreadID = ThreadID;
1713 }
1714 return ThreadID;
1715 }
1716 }
1717 }
1718
1719 // This is not an outlined function region - need to call __kmpc_int32
1720 // kmpc_global_thread_num(ident_t *loc).
1721 // Generate thread id value and cache this value for use across the
1722 // function.
1723 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1724 if (!Elem.second.ServiceInsertPt)
1725 setLocThreadIdInsertPt(CGF);
1726 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1727 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1728 llvm::CallInst *Call = CGF.Builder.CreateCall(
1729 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1730 emitUpdateLocation(CGF, Loc));
1731 Call->setCallingConv(CGF.getRuntimeCC());
1732 Elem.second.ThreadID = Call;
1733 return Call;
1734}
1735
1736void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1737 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1737, __PRETTY_FUNCTION__))
;
1738 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1739 clearLocThreadIdInsertPt(CGF);
1740 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1741 }
1742 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1743 for(auto *D : FunctionUDRMap[CGF.CurFn])
1744 UDRMap.erase(D);
1745 FunctionUDRMap.erase(CGF.CurFn);
1746 }
1747 auto I = FunctionUDMMap.find(CGF.CurFn);
1748 if (I != FunctionUDMMap.end()) {
1749 for(auto *D : I->second)
1750 UDMMap.erase(D);
1751 FunctionUDMMap.erase(I);
1752 }
1753}
1754
1755llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1756 return IdentTy->getPointerTo();
1757}
1758
1759llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1760 if (!Kmpc_MicroTy) {
1761 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1762 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1763 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1764 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1765 }
1766 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1767}
1768
1769llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1770 llvm::FunctionCallee RTLFn = nullptr;
1771 switch (static_cast<OpenMPRTLFunction>(Function)) {
1772 case OMPRTL__kmpc_fork_call: {
1773 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1774 // microtask, ...);
1775 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1776 getKmpc_MicroPointerTy()};
1777 auto *FnTy =
1778 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1779 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1780 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1781 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1782 llvm::LLVMContext &Ctx = F->getContext();
1783 llvm::MDBuilder MDB(Ctx);
1784 // Annotate the callback behavior of the __kmpc_fork_call:
1785 // - The callback callee is argument number 2 (microtask).
1786 // - The first two arguments of the callback callee are unknown (-1).
1787 // - All variadic arguments to the __kmpc_fork_call are passed to the
1788 // callback callee.
1789 F->addMetadata(
1790 llvm::LLVMContext::MD_callback,
1791 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1792 2, {-1, -1},
1793 /* VarArgsArePassed */ true)}));
1794 }
1795 }
1796 break;
1797 }
1798 case OMPRTL__kmpc_global_thread_num: {
1799 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1800 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1801 auto *FnTy =
1802 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1803 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1804 break;
1805 }
1806 case OMPRTL__kmpc_threadprivate_cached: {
1807 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1808 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1809 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1810 CGM.VoidPtrTy, CGM.SizeTy,
1811 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1812 auto *FnTy =
1813 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1814 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1815 break;
1816 }
1817 case OMPRTL__kmpc_critical: {
1818 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1819 // kmp_critical_name *crit);
1820 llvm::Type *TypeParams[] = {
1821 getIdentTyPointerTy(), CGM.Int32Ty,
1822 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1823 auto *FnTy =
1824 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1825 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1826 break;
1827 }
1828 case OMPRTL__kmpc_critical_with_hint: {
1829 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1830 // kmp_critical_name *crit, uintptr_t hint);
1831 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1832 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1833 CGM.IntPtrTy};
1834 auto *FnTy =
1835 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1836 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1837 break;
1838 }
1839 case OMPRTL__kmpc_threadprivate_register: {
1840 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1841 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1842 // typedef void *(*kmpc_ctor)(void *);
1843 auto *KmpcCtorTy =
1844 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1845 /*isVarArg*/ false)->getPointerTo();
1846 // typedef void *(*kmpc_cctor)(void *, void *);
1847 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1848 auto *KmpcCopyCtorTy =
1849 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1850 /*isVarArg*/ false)
1851 ->getPointerTo();
1852 // typedef void (*kmpc_dtor)(void *);
1853 auto *KmpcDtorTy =
1854 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1855 ->getPointerTo();
1856 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1857 KmpcCopyCtorTy, KmpcDtorTy};
1858 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1859 /*isVarArg*/ false);
1860 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1861 break;
1862 }
1863 case OMPRTL__kmpc_end_critical: {
1864 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1865 // kmp_critical_name *crit);
1866 llvm::Type *TypeParams[] = {
1867 getIdentTyPointerTy(), CGM.Int32Ty,
1868 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1869 auto *FnTy =
1870 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1871 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1872 break;
1873 }
1874 case OMPRTL__kmpc_cancel_barrier: {
1875 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1876 // global_tid);
1877 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1878 auto *FnTy =
1879 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1880 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1881 break;
1882 }
1883 case OMPRTL__kmpc_barrier: {
1884 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1885 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1886 auto *FnTy =
1887 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1888 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1889 break;
1890 }
1891 case OMPRTL__kmpc_for_static_fini: {
1892 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1893 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1894 auto *FnTy =
1895 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1896 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1897 break;
1898 }
1899 case OMPRTL__kmpc_push_num_threads: {
1900 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1901 // kmp_int32 num_threads)
1902 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1903 CGM.Int32Ty};
1904 auto *FnTy =
1905 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1906 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1907 break;
1908 }
1909 case OMPRTL__kmpc_serialized_parallel: {
1910 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1911 // global_tid);
1912 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1913 auto *FnTy =
1914 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1915 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1916 break;
1917 }
1918 case OMPRTL__kmpc_end_serialized_parallel: {
1919 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1920 // global_tid);
1921 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1922 auto *FnTy =
1923 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1924 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1925 break;
1926 }
1927 case OMPRTL__kmpc_flush: {
1928 // Build void __kmpc_flush(ident_t *loc);
1929 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1930 auto *FnTy =
1931 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1932 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1933 break;
1934 }
1935 case OMPRTL__kmpc_master: {
1936 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1937 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1938 auto *FnTy =
1939 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1940 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1941 break;
1942 }
1943 case OMPRTL__kmpc_end_master: {
1944 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1945 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1946 auto *FnTy =
1947 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1948 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1949 break;
1950 }
1951 case OMPRTL__kmpc_omp_taskyield: {
1952 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1953 // int end_part);
1954 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1955 auto *FnTy =
1956 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1957 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1958 break;
1959 }
1960 case OMPRTL__kmpc_single: {
1961 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1962 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1963 auto *FnTy =
1964 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1965 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1966 break;
1967 }
1968 case OMPRTL__kmpc_end_single: {
1969 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1970 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1971 auto *FnTy =
1972 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1973 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1974 break;
1975 }
1976 case OMPRTL__kmpc_omp_task_alloc: {
1977 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1978 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1979 // kmp_routine_entry_t *task_entry);
1980 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1981, __PRETTY_FUNCTION__))
1981 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1981, __PRETTY_FUNCTION__))
;
1982 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1983 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1984 // Return void * and then cast to particular kmp_task_t type.
1985 auto *FnTy =
1986 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1987 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1988 break;
1989 }
1990 case OMPRTL__kmpc_omp_target_task_alloc: {
1991 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
1992 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1993 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
1994 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1995, __PRETTY_FUNCTION__))
1995 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1995, __PRETTY_FUNCTION__))
;
1996 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1997 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
1998 CGM.Int64Ty};
1999 // Return void * and then cast to particular kmp_task_t type.
2000 auto *FnTy =
2001 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2002 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2003 break;
2004 }
2005 case OMPRTL__kmpc_omp_task: {
2006 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2007 // *new_task);
2008 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2009 CGM.VoidPtrTy};
2010 auto *FnTy =
2011 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2012 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2013 break;
2014 }
2015 case OMPRTL__kmpc_copyprivate: {
2016 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2017 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2018 // kmp_int32 didit);
2019 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2020 auto *CpyFnTy =
2021 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2022 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2023 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2024 CGM.Int32Ty};
2025 auto *FnTy =
2026 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2027 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2028 break;
2029 }
2030 case OMPRTL__kmpc_reduce: {
2031 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2032 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2033 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2034 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2035 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2036 /*isVarArg=*/false);
2037 llvm::Type *TypeParams[] = {
2038 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2039 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2040 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2041 auto *FnTy =
2042 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2043 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2044 break;
2045 }
2046 case OMPRTL__kmpc_reduce_nowait: {
2047 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2048 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2049 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2050 // *lck);
2051 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2052 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2053 /*isVarArg=*/false);
2054 llvm::Type *TypeParams[] = {
2055 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2056 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2057 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2058 auto *FnTy =
2059 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2060 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2061 break;
2062 }
2063 case OMPRTL__kmpc_end_reduce: {
2064 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2065 // kmp_critical_name *lck);
2066 llvm::Type *TypeParams[] = {
2067 getIdentTyPointerTy(), CGM.Int32Ty,
2068 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2069 auto *FnTy =
2070 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2071 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2072 break;
2073 }
2074 case OMPRTL__kmpc_end_reduce_nowait: {
2075 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2076 // kmp_critical_name *lck);
2077 llvm::Type *TypeParams[] = {
2078 getIdentTyPointerTy(), CGM.Int32Ty,
2079 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2080 auto *FnTy =
2081 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2082 RTLFn =
2083 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2084 break;
2085 }
2086 case OMPRTL__kmpc_omp_task_begin_if0: {
2087 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2088 // *new_task);
2089 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2090 CGM.VoidPtrTy};
2091 auto *FnTy =
2092 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2093 RTLFn =
2094 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2095 break;
2096 }
2097 case OMPRTL__kmpc_omp_task_complete_if0: {
2098 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2099 // *new_task);
2100 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2101 CGM.VoidPtrTy};
2102 auto *FnTy =
2103 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2104 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2105 /*Name=*/"__kmpc_omp_task_complete_if0");
2106 break;
2107 }
2108 case OMPRTL__kmpc_ordered: {
2109 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2110 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2111 auto *FnTy =
2112 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2113 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2114 break;
2115 }
2116 case OMPRTL__kmpc_end_ordered: {
2117 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2118 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2119 auto *FnTy =
2120 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2121 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2122 break;
2123 }
2124 case OMPRTL__kmpc_omp_taskwait: {
2125 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2126 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2127 auto *FnTy =
2128 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2129 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2130 break;
2131 }
2132 case OMPRTL__kmpc_taskgroup: {
2133 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2134 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2135 auto *FnTy =
2136 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2137 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2138 break;
2139 }
2140 case OMPRTL__kmpc_end_taskgroup: {
2141 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2142 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2143 auto *FnTy =
2144 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2145 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2146 break;
2147 }
2148 case OMPRTL__kmpc_push_proc_bind: {
2149 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2150 // int proc_bind)
2151 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2152 auto *FnTy =
2153 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2154 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2155 break;
2156 }
2157 case OMPRTL__kmpc_omp_task_with_deps: {
2158 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2159 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2160 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2161 llvm::Type *TypeParams[] = {
2162 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2163 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2164 auto *FnTy =
2165 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2166 RTLFn =
2167 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2168 break;
2169 }
2170 case OMPRTL__kmpc_omp_wait_deps: {
2171 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2172 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2173 // kmp_depend_info_t *noalias_dep_list);
2174 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2175 CGM.Int32Ty, CGM.VoidPtrTy,
2176 CGM.Int32Ty, CGM.VoidPtrTy};
2177 auto *FnTy =
2178 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2179 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2180 break;
2181 }
2182 case OMPRTL__kmpc_cancellationpoint: {
2183 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2184 // global_tid, kmp_int32 cncl_kind)
2185 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2186 auto *FnTy =
2187 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2188 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2189 break;
2190 }
2191 case OMPRTL__kmpc_cancel: {
2192 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2193 // kmp_int32 cncl_kind)
2194 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2195 auto *FnTy =
2196 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2197 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2198 break;
2199 }
2200 case OMPRTL__kmpc_push_num_teams: {
2201 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2202 // kmp_int32 num_teams, kmp_int32 num_threads)
2203 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2204 CGM.Int32Ty};
2205 auto *FnTy =
2206 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2207 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2208 break;
2209 }
2210 case OMPRTL__kmpc_fork_teams: {
2211 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2212 // microtask, ...);
2213 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2214 getKmpc_MicroPointerTy()};
2215 auto *FnTy =
2216 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2217 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2218 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2219 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2220 llvm::LLVMContext &Ctx = F->getContext();
2221 llvm::MDBuilder MDB(Ctx);
2222 // Annotate the callback behavior of the __kmpc_fork_teams:
2223 // - The callback callee is argument number 2 (microtask).
2224 // - The first two arguments of the callback callee are unknown (-1).
2225 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2226 // callback callee.
2227 F->addMetadata(
2228 llvm::LLVMContext::MD_callback,
2229 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2230 2, {-1, -1},
2231 /* VarArgsArePassed */ true)}));
2232 }
2233 }
2234 break;
2235 }
2236 case OMPRTL__kmpc_taskloop: {
2237 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2238 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2239 // sched, kmp_uint64 grainsize, void *task_dup);
2240 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2241 CGM.IntTy,
2242 CGM.VoidPtrTy,
2243 CGM.IntTy,
2244 CGM.Int64Ty->getPointerTo(),
2245 CGM.Int64Ty->getPointerTo(),
2246 CGM.Int64Ty,
2247 CGM.IntTy,
2248 CGM.IntTy,
2249 CGM.Int64Ty,
2250 CGM.VoidPtrTy};
2251 auto *FnTy =
2252 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2253 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2254 break;
2255 }
2256 case OMPRTL__kmpc_doacross_init: {
2257 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2258 // num_dims, struct kmp_dim *dims);
2259 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2260 CGM.Int32Ty,
2261 CGM.Int32Ty,
2262 CGM.VoidPtrTy};
2263 auto *FnTy =
2264 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2265 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2266 break;
2267 }
2268 case OMPRTL__kmpc_doacross_fini: {
2269 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2270 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2271 auto *FnTy =
2272 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2273 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2274 break;
2275 }
2276 case OMPRTL__kmpc_doacross_post: {
2277 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2278 // *vec);
2279 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2280 CGM.Int64Ty->getPointerTo()};
2281 auto *FnTy =
2282 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2283 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2284 break;
2285 }
2286 case OMPRTL__kmpc_doacross_wait: {
2287 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2288 // *vec);
2289 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2290 CGM.Int64Ty->getPointerTo()};
2291 auto *FnTy =
2292 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2293 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2294 break;
2295 }
2296 case OMPRTL__kmpc_task_reduction_init: {
2297 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2298 // *data);
2299 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2300 auto *FnTy =
2301 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2302 RTLFn =
2303 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2304 break;
2305 }
2306 case OMPRTL__kmpc_task_reduction_get_th_data: {
2307 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2308 // *d);
2309 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2310 auto *FnTy =
2311 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2312 RTLFn = CGM.CreateRuntimeFunction(
2313 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2314 break;
2315 }
2316 case OMPRTL__kmpc_alloc: {
2317 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2318 // al); omp_allocator_handle_t type is void *.
2319 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2320 auto *FnTy =
2321 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2322 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2323 break;
2324 }
2325 case OMPRTL__kmpc_free: {
2326 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2327 // al); omp_allocator_handle_t type is void *.
2328 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2329 auto *FnTy =
2330 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2331 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2332 break;
2333 }
2334 case OMPRTL__kmpc_push_target_tripcount: {
2335 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2336 // size);
2337 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2338 llvm::FunctionType *FnTy =
2339 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2340 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2341 break;
2342 }
2343 case OMPRTL__tgt_target: {
2344 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2345 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2346 // *arg_types);
2347 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2348 CGM.VoidPtrTy,
2349 CGM.Int32Ty,
2350 CGM.VoidPtrPtrTy,
2351 CGM.VoidPtrPtrTy,
2352 CGM.Int64Ty->getPointerTo(),
2353 CGM.Int64Ty->getPointerTo()};
2354 auto *FnTy =
2355 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2356 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2357 break;
2358 }
2359 case OMPRTL__tgt_target_nowait: {
2360 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2361 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2362 // int64_t *arg_types);
2363 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2364 CGM.VoidPtrTy,
2365 CGM.Int32Ty,
2366 CGM.VoidPtrPtrTy,
2367 CGM.VoidPtrPtrTy,
2368 CGM.Int64Ty->getPointerTo(),
2369 CGM.Int64Ty->getPointerTo()};
2370 auto *FnTy =
2371 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2372 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2373 break;
2374 }
2375 case OMPRTL__tgt_target_teams: {
2376 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2377 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2378 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2379 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2380 CGM.VoidPtrTy,
2381 CGM.Int32Ty,
2382 CGM.VoidPtrPtrTy,
2383 CGM.VoidPtrPtrTy,
2384 CGM.Int64Ty->getPointerTo(),
2385 CGM.Int64Ty->getPointerTo(),
2386 CGM.Int32Ty,
2387 CGM.Int32Ty};
2388 auto *FnTy =
2389 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2390 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2391 break;
2392 }
2393 case OMPRTL__tgt_target_teams_nowait: {
2394 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2395 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2396 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2397 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2398 CGM.VoidPtrTy,
2399 CGM.Int32Ty,
2400 CGM.VoidPtrPtrTy,
2401 CGM.VoidPtrPtrTy,
2402 CGM.Int64Ty->getPointerTo(),
2403 CGM.Int64Ty->getPointerTo(),
2404 CGM.Int32Ty,
2405 CGM.Int32Ty};
2406 auto *FnTy =
2407 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2408 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2409 break;
2410 }
2411 case OMPRTL__tgt_register_requires: {
2412 // Build void __tgt_register_requires(int64_t flags);
2413 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2414 auto *FnTy =
2415 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2416 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2417 break;
2418 }
2419 case OMPRTL__tgt_register_lib: {
2420 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2421 QualType ParamTy =
2422 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2423 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2424 auto *FnTy =
2425 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2426 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2427 break;
2428 }
2429 case OMPRTL__tgt_unregister_lib: {
2430 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2431 QualType ParamTy =
2432 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2433 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2434 auto *FnTy =
2435 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2436 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2437 break;
2438 }
2439 case OMPRTL__tgt_target_data_begin: {
2440 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2441 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2442 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2443 CGM.Int32Ty,
2444 CGM.VoidPtrPtrTy,
2445 CGM.VoidPtrPtrTy,
2446 CGM.Int64Ty->getPointerTo(),
2447 CGM.Int64Ty->getPointerTo()};
2448 auto *FnTy =
2449 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2450 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2451 break;
2452 }
2453 case OMPRTL__tgt_target_data_begin_nowait: {
2454 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2455 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2456 // *arg_types);
2457 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2458 CGM.Int32Ty,
2459 CGM.VoidPtrPtrTy,
2460 CGM.VoidPtrPtrTy,
2461 CGM.Int64Ty->getPointerTo(),
2462 CGM.Int64Ty->getPointerTo()};
2463 auto *FnTy =
2464 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2465 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2466 break;
2467 }
2468 case OMPRTL__tgt_target_data_end: {
2469 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2470 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2471 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2472 CGM.Int32Ty,
2473 CGM.VoidPtrPtrTy,
2474 CGM.VoidPtrPtrTy,
2475 CGM.Int64Ty->getPointerTo(),
2476 CGM.Int64Ty->getPointerTo()};
2477 auto *FnTy =
2478 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2479 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2480 break;
2481 }
2482 case OMPRTL__tgt_target_data_end_nowait: {
2483 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2484 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2485 // *arg_types);
2486 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2487 CGM.Int32Ty,
2488 CGM.VoidPtrPtrTy,
2489 CGM.VoidPtrPtrTy,
2490 CGM.Int64Ty->getPointerTo(),
2491 CGM.Int64Ty->getPointerTo()};
2492 auto *FnTy =
2493 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2494 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2495 break;
2496 }
2497 case OMPRTL__tgt_target_data_update: {
2498 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2499 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2500 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2501 CGM.Int32Ty,
2502 CGM.VoidPtrPtrTy,
2503 CGM.VoidPtrPtrTy,
2504 CGM.Int64Ty->getPointerTo(),
2505 CGM.Int64Ty->getPointerTo()};
2506 auto *FnTy =
2507 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2508 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2509 break;
2510 }
2511 case OMPRTL__tgt_target_data_update_nowait: {
2512 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2513 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2514 // *arg_types);
2515 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2516 CGM.Int32Ty,
2517 CGM.VoidPtrPtrTy,
2518 CGM.VoidPtrPtrTy,
2519 CGM.Int64Ty->getPointerTo(),
2520 CGM.Int64Ty->getPointerTo()};
2521 auto *FnTy =
2522 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2523 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2524 break;
2525 }
2526 case OMPRTL__tgt_mapper_num_components: {
2527 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2528 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2529 auto *FnTy =
2530 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2531 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2532 break;
2533 }
2534 case OMPRTL__tgt_push_mapper_component: {
2535 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2536 // *base, void *begin, int64_t size, int64_t type);
2537 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2538 CGM.Int64Ty, CGM.Int64Ty};
2539 auto *FnTy =
2540 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2541 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2542 break;
2543 }
2544 }
2545 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2545, __PRETTY_FUNCTION__))
;
2546 return RTLFn;
2547}
2548
2549llvm::FunctionCallee
2550CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2551 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2552, __PRETTY_FUNCTION__))
2552 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2552, __PRETTY_FUNCTION__))
;
2553 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2554 : "__kmpc_for_static_init_4u")
2555 : (IVSigned ? "__kmpc_for_static_init_8"
2556 : "__kmpc_for_static_init_8u");
2557 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2558 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2559 llvm::Type *TypeParams[] = {
2560 getIdentTyPointerTy(), // loc
2561 CGM.Int32Ty, // tid
2562 CGM.Int32Ty, // schedtype
2563 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2564 PtrTy, // p_lower
2565 PtrTy, // p_upper
2566 PtrTy, // p_stride
2567 ITy, // incr
2568 ITy // chunk
2569 };
2570 auto *FnTy =
2571 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2572 return CGM.CreateRuntimeFunction(FnTy, Name);
2573}
2574
2575llvm::FunctionCallee
2576CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2577 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2578, __PRETTY_FUNCTION__))
2578 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2578, __PRETTY_FUNCTION__))
;
2579 StringRef Name =
2580 IVSize == 32
2581 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2582 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2583 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2584 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2585 CGM.Int32Ty, // tid
2586 CGM.Int32Ty, // schedtype
2587 ITy, // lower
2588 ITy, // upper
2589 ITy, // stride
2590 ITy // chunk
2591 };
2592 auto *FnTy =
2593 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2594 return CGM.CreateRuntimeFunction(FnTy, Name);
2595}
2596
2597llvm::FunctionCallee
2598CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2599 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2600, __PRETTY_FUNCTION__))
2600 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2600, __PRETTY_FUNCTION__))
;
2601 StringRef Name =
2602 IVSize == 32
2603 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2604 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2605 llvm::Type *TypeParams[] = {
2606 getIdentTyPointerTy(), // loc
2607 CGM.Int32Ty, // tid
2608 };
2609 auto *FnTy =
2610 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2611 return CGM.CreateRuntimeFunction(FnTy, Name);
2612}
2613
2614llvm::FunctionCallee
2615CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2616 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2617, __PRETTY_FUNCTION__))
2617 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2617, __PRETTY_FUNCTION__))
;
2618 StringRef Name =
2619 IVSize == 32
2620 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2621 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2622 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2623 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2624 llvm::Type *TypeParams[] = {
2625 getIdentTyPointerTy(), // loc
2626 CGM.Int32Ty, // tid
2627 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2628 PtrTy, // p_lower
2629 PtrTy, // p_upper
2630 PtrTy // p_stride
2631 };
2632 auto *FnTy =
2633 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2634 return CGM.CreateRuntimeFunction(FnTy, Name);
2635}
2636
2637/// Obtain information that uniquely identifies a target entry. This
2638/// consists of the file and device IDs as well as line number associated with
2639/// the relevant entry source location.
2640static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2641 unsigned &DeviceID, unsigned &FileID,
2642 unsigned &LineNum) {
2643 SourceManager &SM = C.getSourceManager();
2644
2645 // The loc should be always valid and have a file ID (the user cannot use
2646 // #pragma directives in macros)
2647
2648 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2648, __PRETTY_FUNCTION__))
;
2649
2650 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2651 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2651, __PRETTY_FUNCTION__))
;
2652
2653 llvm::sys::fs::UniqueID ID;
2654 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2655 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2656 << PLoc.getFilename() << EC.message();
2657
2658 DeviceID = ID.getDevice();
2659 FileID = ID.getFile();
2660 LineNum = PLoc.getLine();
2661}
2662
2663Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2664 if (CGM.getLangOpts().OpenMPSimd)
2665 return Address::invalid();
2666 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2667 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2668 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2669 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2670 HasRequiresUnifiedSharedMemory))) {
2671 SmallString<64> PtrName;
2672 {
2673 llvm::raw_svector_ostream OS(PtrName);
2674 OS << CGM.getMangledName(GlobalDecl(VD));
2675 if (!VD->isExternallyVisible()) {
2676 unsigned DeviceID, FileID, Line;
2677 getTargetEntryUniqueInfo(CGM.getContext(),
2678 VD->getCanonicalDecl()->getBeginLoc(),
2679 DeviceID, FileID, Line);
2680 OS << llvm::format("_%x", FileID);
2681 }
2682 OS << "_decl_tgt_ref_ptr";
2683 }
2684 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2685 if (!Ptr) {
2686 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2687 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2688 PtrName);
2689
2690 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2691 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2692
2693 if (!CGM.getLangOpts().OpenMPIsDevice)
2694 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2695 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2696 }
2697 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2698 }
2699 return Address::invalid();
2700}
2701
2702llvm::Constant *
2703CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2704 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2705, __PRETTY_FUNCTION__))
2705 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2705, __PRETTY_FUNCTION__))
;
2706 // Lookup the entry, lazily creating it if necessary.
2707 std::string Suffix = getName({"cache", ""});
2708 return getOrCreateInternalVariable(
2709 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2710}
2711
2712Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2713 const VarDecl *VD,
2714 Address VDAddr,
2715 SourceLocation Loc) {
2716 if (CGM.getLangOpts().OpenMPUseTLS &&
2717 CGM.getContext().getTargetInfo().isTLSSupported())
2718 return VDAddr;
2719
2720 llvm::Type *VarTy = VDAddr.getElementType();
2721 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2722 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2723 CGM.Int8PtrTy),
2724 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2725 getOrCreateThreadPrivateCache(VD)};
2726 return Address(CGF.EmitRuntimeCall(
2727 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2728 VDAddr.getAlignment());
2729}
2730
2731void CGOpenMPRuntime::emitThreadPrivateVarInit(
2732 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2733 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2734 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2735 // library.
2736 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2737 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2738 OMPLoc);
2739 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2740 // to register constructor/destructor for variable.
2741 llvm::Value *Args[] = {
2742 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2743 Ctor, CopyCtor, Dtor};
2744 CGF.EmitRuntimeCall(
2745 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2746}
2747
2748llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2749 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2750 bool PerformInit, CodeGenFunction *CGF) {
2751 if (CGM.getLangOpts().OpenMPUseTLS &&
2752 CGM.getContext().getTargetInfo().isTLSSupported())
2753 return nullptr;
2754
2755 VD = VD->getDefinition(CGM.getContext());
2756 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2757 QualType ASTTy = VD->getType();
2758
2759 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2760 const Expr *Init = VD->getAnyInitializer();
2761 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2762 // Generate function that re-emits the declaration's initializer into the
2763 // threadprivate copy of the variable VD
2764 CodeGenFunction CtorCGF(CGM);
2765 FunctionArgList Args;
2766 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2767 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2768 ImplicitParamDecl::Other);
2769 Args.push_back(&Dst);
2770
2771 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2772 CGM.getContext().VoidPtrTy, Args);
2773 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2774 std::string Name = getName({"__kmpc_global_ctor_", ""});
2775 llvm::Function *Fn =
2776 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2777 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2778 Args, Loc, Loc);
2779 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2780 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2781 CGM.getContext().VoidPtrTy, Dst.getLocation());
2782 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2783 Arg = CtorCGF.Builder.CreateElementBitCast(
2784 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2785 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2786 /*IsInitializer=*/true);
2787 ArgVal = CtorCGF.EmitLoadOfScalar(
2788 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2789 CGM.getContext().VoidPtrTy, Dst.getLocation());
2790 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2791 CtorCGF.FinishFunction();
2792 Ctor = Fn;
2793 }
2794 if (VD->getType().isDestructedType() != QualType::DK_none) {
2795 // Generate function that emits destructor call for the threadprivate copy
2796 // of the variable VD
2797 CodeGenFunction DtorCGF(CGM);
2798 FunctionArgList Args;
2799 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2800 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2801 ImplicitParamDecl::Other);
2802 Args.push_back(&Dst);
2803
2804 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2805 CGM.getContext().VoidTy, Args);
2806 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2807 std::string Name = getName({"__kmpc_global_dtor_", ""});
2808 llvm::Function *Fn =
2809 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2810 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2811 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2812 Loc, Loc);
2813 // Create a scope with an artificial location for the body of this function.
2814 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2815 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2816 DtorCGF.GetAddrOfLocalVar(&Dst),
2817 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2818 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2819 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2820 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2821 DtorCGF.FinishFunction();
2822 Dtor = Fn;
2823 }
2824 // Do not emit init function if it is not required.
2825 if (!Ctor && !Dtor)
2826 return nullptr;
2827
2828 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2829 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2830 /*isVarArg=*/false)
2831 ->getPointerTo();
2832 // Copying constructor for the threadprivate variable.
2833 // Must be NULL - reserved by runtime, but currently it requires that this
2834 // parameter is always NULL. Otherwise it fires assertion.
2835 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2836 if (Ctor == nullptr) {
2837 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2838 /*isVarArg=*/false)
2839 ->getPointerTo();
2840 Ctor = llvm::Constant::getNullValue(CtorTy);
2841 }
2842 if (Dtor == nullptr) {
2843 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2844 /*isVarArg=*/false)
2845 ->getPointerTo();
2846 Dtor = llvm::Constant::getNullValue(DtorTy);
2847 }
2848 if (!CGF) {
2849 auto *InitFunctionTy =
2850 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2851 std::string Name = getName({"__omp_threadprivate_init_", ""});
2852 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2853 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2854 CodeGenFunction InitCGF(CGM);
2855 FunctionArgList ArgList;
2856 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2857 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2858 Loc, Loc);
2859 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2860 InitCGF.FinishFunction();
2861 return InitFunction;
2862 }
2863 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2864 }
2865 return nullptr;
2866}
2867
2868bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2869 llvm::GlobalVariable *Addr,
2870 bool PerformInit) {
2871 if (CGM.getLangOpts().OMPTargetTriples.empty())
1
Assuming the condition is false
2
Taking false branch
2872 return false;
2873 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2874 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2875 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
3
Calling 'Optional::operator bool'
11
Returning from 'Optional::operator bool'
12
Assuming the condition is false
2876 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
13
Assuming the condition is false
2877 HasRequiresUnifiedSharedMemory))
2878 return CGM.getLangOpts().OpenMPIsDevice;
2879 VD = VD->getDefinition(CGM.getContext());
14
Value assigned to 'VD'
2880 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
15
Assuming 'VD' is null
16
Taking false branch
2881 return CGM.getLangOpts().OpenMPIsDevice;
2882
2883 QualType ASTTy = VD->getType();
17
Called C++ object pointer is null
2884
2885 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2886 // Produce the unique prefix to identify the new target regions. We use
2887 // the source location of the variable declaration which we know to not
2888 // conflict with any target region.
2889 unsigned DeviceID;
2890 unsigned FileID;
2891 unsigned Line;
2892 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2893 SmallString<128> Buffer, Out;
2894 {
2895 llvm::raw_svector_ostream OS(Buffer);
2896 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2897 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2898 }
2899
2900 const Expr *Init = VD->getAnyInitializer();
2901 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2902 llvm::Constant *Ctor;
2903 llvm::Constant *ID;
2904 if (CGM.getLangOpts().OpenMPIsDevice) {
2905 // Generate function that re-emits the declaration's initializer into
2906 // the threadprivate copy of the variable VD
2907 CodeGenFunction CtorCGF(CGM);
2908
2909 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2910 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2911 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2912 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2913 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2914 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2915 FunctionArgList(), Loc, Loc);
2916 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2917 CtorCGF.EmitAnyExprToMem(Init,
2918 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2919 Init->getType().getQualifiers(),
2920 /*IsInitializer=*/true);
2921 CtorCGF.FinishFunction();
2922 Ctor = Fn;
2923 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2924 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2925 } else {
2926 Ctor = new llvm::GlobalVariable(
2927 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2928 llvm::GlobalValue::PrivateLinkage,
2929 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2930 ID = Ctor;
2931 }
2932
2933 // Register the information for the entry associated with the constructor.
2934 Out.clear();
2935 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2936 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2937 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2938 }
2939 if (VD->getType().isDestructedType() != QualType::DK_none) {
2940 llvm::Constant *Dtor;
2941 llvm::Constant *ID;
2942 if (CGM.getLangOpts().OpenMPIsDevice) {
2943 // Generate function that emits destructor call for the threadprivate
2944 // copy of the variable VD
2945 CodeGenFunction DtorCGF(CGM);
2946
2947 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2948 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2949 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2950 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2951 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2952 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2953 FunctionArgList(), Loc, Loc);
2954 // Create a scope with an artificial location for the body of this
2955 // function.
2956 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2957 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2958 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2959 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2960 DtorCGF.FinishFunction();
2961 Dtor = Fn;
2962 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2963 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2964 } else {
2965 Dtor = new llvm::GlobalVariable(
2966 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2967 llvm::GlobalValue::PrivateLinkage,
2968 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2969 ID = Dtor;
2970 }
2971 // Register the information for the entry associated with the destructor.
2972 Out.clear();
2973 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2974 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2975 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2976 }
2977 return CGM.getLangOpts().OpenMPIsDevice;
2978}
2979
2980Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2981 QualType VarType,
2982 StringRef Name) {
2983 std::string Suffix = getName({"artificial", ""});
2984 std::string CacheSuffix = getName({"cache", ""});
2985 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2986 llvm::Value *GAddr =
2987 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2988 llvm::Value *Args[] = {
2989 emitUpdateLocation(CGF, SourceLocation()),
2990 getThreadID(CGF, SourceLocation()),
2991 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2992 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2993 /*isSigned=*/false),
2994 getOrCreateInternalVariable(
2995 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2996 return Address(
2997 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2998 CGF.EmitRuntimeCall(
2999 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3000 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3001 CGM.getPointerAlign());
3002}
3003
3004void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
3005 const RegionCodeGenTy &ThenGen,
3006 const RegionCodeGenTy &ElseGen) {
3007 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3008
3009 // If the condition constant folds and can be elided, try to avoid emitting
3010 // the condition and the dead arm of the if/else.
3011 bool CondConstant;
3012 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
3013 if (CondConstant)
3014 ThenGen(CGF);
3015 else
3016 ElseGen(CGF);
3017 return;
3018 }
3019
3020 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3021 // emit the conditional branch.
3022 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3023 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3024 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3025 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3026
3027 // Emit the 'then' code.
3028 CGF.EmitBlock(ThenBlock);
3029 ThenGen(CGF);
3030 CGF.EmitBranch(ContBlock);
3031 // Emit the 'else' code if present.
3032 // There is no need to emit line number for unconditional branch.
3033 (void)ApplyDebugLocation::CreateEmpty(CGF);
3034 CGF.EmitBlock(ElseBlock);
3035 ElseGen(CGF);
3036 // There is no need to emit line number for unconditional branch.
3037 (void)ApplyDebugLocation::CreateEmpty(CGF);
3038 CGF.EmitBranch(ContBlock);
3039 // Emit the continuation block for code after the if.
3040 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3041}
3042
3043void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3044 llvm::Function *OutlinedFn,
3045 ArrayRef<llvm::Value *> CapturedVars,
3046 const Expr *IfCond) {
3047 if (!CGF.HaveInsertPoint())
3048 return;
3049 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3050 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3051 PrePostActionTy &) {
3052 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3053 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3054 llvm::Value *Args[] = {
3055 RTLoc,
3056 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3057 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3058 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3059 RealArgs.append(std::begin(Args), std::end(Args));
3060 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3061
3062 llvm::FunctionCallee RTLFn =
3063 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3064 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3065 };
3066 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3067 PrePostActionTy &) {
3068 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3069 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3070 // Build calls:
3071 // __kmpc_serialized_parallel(&Loc, GTid);
3072 llvm::Value *Args[] = {RTLoc, ThreadID};
3073 CGF.EmitRuntimeCall(
3074 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3075
3076 // OutlinedFn(&GTid, &zero, CapturedStruct);
3077 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3078 /*Name*/ ".zero.addr");
3079 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
3080 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3081 // ThreadId for serialized parallels is 0.
3082 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3083 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3084 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3085 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3086
3087 // __kmpc_end_serialized_parallel(&Loc, GTid);
3088 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3089 CGF.EmitRuntimeCall(
3090 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3091 EndArgs);
3092 };
3093 if (IfCond) {
3094 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
3095 } else {
3096 RegionCodeGenTy ThenRCG(ThenGen);
3097 ThenRCG(CGF);
3098 }
3099}
3100
3101// If we're inside an (outlined) parallel region, use the region info's
3102// thread-ID variable (it is passed in a first argument of the outlined function
3103// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3104// regular serial code region, get thread ID by calling kmp_int32
3105// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3106// return the address of that temp.
3107Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3108 SourceLocation Loc) {
3109 if (auto *OMPRegionInfo =
3110 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3111 if (OMPRegionInfo->getThreadIDVariable())
3112 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
3113
3114 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3115 QualType Int32Ty =
3116 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3117 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3118 CGF.EmitStoreOfScalar(ThreadID,
3119 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3120
3121 return ThreadIDTemp;
3122}
3123
3124llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3125 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3126 SmallString<256> Buffer;
3127 llvm::raw_svector_ostream Out(Buffer);
3128 Out << Name;
3129 StringRef RuntimeName = Out.str();
3130 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3131 if (Elem.second) {
3132 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3133, __PRETTY_FUNCTION__))
3133 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3133, __PRETTY_FUNCTION__))
;
3134 return &*Elem.second;
3135 }
3136
3137 return Elem.second = new llvm::GlobalVariable(
3138 CGM.getModule(), Ty, /*IsConstant*/ false,
3139 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3140 Elem.first(), /*InsertBefore=*/nullptr,
3141 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3142}
3143
3144llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3145 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3146 std::string Name = getName({Prefix, "var"});
3147 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3148}
3149
3150namespace {
3151/// Common pre(post)-action for different OpenMP constructs.
3152class CommonActionTy final : public PrePostActionTy {
3153 llvm::FunctionCallee EnterCallee;
3154 ArrayRef<llvm::Value *> EnterArgs;
3155 llvm::FunctionCallee ExitCallee;
3156 ArrayRef<llvm::Value *> ExitArgs;
3157 bool Conditional;
3158 llvm::BasicBlock *ContBlock = nullptr;
3159
3160public:
3161 CommonActionTy(llvm::FunctionCallee EnterCallee,
3162 ArrayRef<llvm::Value *> EnterArgs,
3163 llvm::FunctionCallee ExitCallee,
3164 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3165 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3166 ExitArgs(ExitArgs), Conditional(Conditional) {}
3167 void Enter(CodeGenFunction &CGF) override {
3168 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3169 if (Conditional) {
3170 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3171 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3172 ContBlock = CGF.createBasicBlock("omp_if.end");
3173 // Generate the branch (If-stmt)
3174 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3175 CGF.EmitBlock(ThenBlock);
3176 }
3177 }
3178 void Done(CodeGenFunction &CGF) {
3179 // Emit the rest of blocks/branches
3180 CGF.EmitBranch(ContBlock);
3181 CGF.EmitBlock(ContBlock, true);
3182 }
3183 void Exit(CodeGenFunction &CGF) override {
3184 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3185 }
3186};
3187} // anonymous namespace
3188
3189void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3190 StringRef CriticalName,
3191 const RegionCodeGenTy &CriticalOpGen,
3192 SourceLocation Loc, const Expr *Hint) {
3193 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3194 // CriticalOpGen();
3195 // __kmpc_end_critical(ident_t *, gtid, Lock);
3196 // Prepare arguments and build a call to __kmpc_critical
3197 if (!CGF.HaveInsertPoint())
3198 return;
3199 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3200 getCriticalRegionLock(CriticalName)};
3201 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3202 std::end(Args));
3203 if (Hint) {
3204 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3205 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3206 }
3207 CommonActionTy Action(
3208 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3209 : OMPRTL__kmpc_critical),
3210 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3211 CriticalOpGen.setAction(Action);
3212 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3213}
3214
3215void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3216 const RegionCodeGenTy &MasterOpGen,
3217 SourceLocation Loc) {
3218 if (!CGF.HaveInsertPoint())
3219 return;
3220 // if(__kmpc_master(ident_t *, gtid)) {
3221 // MasterOpGen();
3222 // __kmpc_end_master(ident_t *, gtid);
3223 // }
3224 // Prepare arguments and build a call to __kmpc_master
3225 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3226 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3227 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3228 /*Conditional=*/true);
3229 MasterOpGen.setAction(Action);
3230 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3231 Action.Done(CGF);
3232}
3233
3234void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3235 SourceLocation Loc) {
3236 if (!CGF.HaveInsertPoint())
3237 return;
3238 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3239 llvm::Value *Args[] = {
3240 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3241 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3242 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3243 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3244 Region->emitUntiedSwitch(CGF);
3245}
3246
3247void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3248 const RegionCodeGenTy &TaskgroupOpGen,
3249 SourceLocation Loc) {
3250 if (!CGF.HaveInsertPoint())
3251 return;
3252 // __kmpc_taskgroup(ident_t *, gtid);
3253 // TaskgroupOpGen();
3254 // __kmpc_end_taskgroup(ident_t *, gtid);
3255 // Prepare arguments and build a call to __kmpc_taskgroup
3256 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3257 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3258 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3259 Args);
3260 TaskgroupOpGen.setAction(Action);
3261 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3262}
3263
3264/// Given an array of pointers to variables, project the address of a
3265/// given variable.
3266static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3267 unsigned Index, const VarDecl *Var) {
3268 // Pull out the pointer to the variable.
3269 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3270 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3271
3272 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3273 Addr = CGF.Builder.CreateElementBitCast(
3274 Addr, CGF.ConvertTypeForMem(Var->getType()));
3275 return Addr;
3276}
3277
3278static llvm::Value *emitCopyprivateCopyFunction(
3279 CodeGenModule &CGM, llvm::Type *ArgsType,
3280 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3281 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3282 SourceLocation Loc) {
3283 ASTContext &C = CGM.getContext();
3284 // void copy_func(void *LHSArg, void *RHSArg);
3285 FunctionArgList Args;
3286 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3287 ImplicitParamDecl::Other);
3288 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3289 ImplicitParamDecl::Other);
3290 Args.push_back(&LHSArg);
3291 Args.push_back(&RHSArg);
3292 const auto &CGFI =
3293 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3294 std::string Name =
3295 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3296 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3297 llvm::GlobalValue::InternalLinkage, Name,
3298 &CGM.getModule());
3299 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3300 Fn->setDoesNotRecurse();
3301 CodeGenFunction CGF(CGM);
3302 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3303 // Dest = (void*[n])(LHSArg);
3304 // Src = (void*[n])(RHSArg);
3305 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3306 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3307 ArgsType), CGF.getPointerAlign());
3308 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3309 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3310 ArgsType), CGF.getPointerAlign());
3311 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3312 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3313 // ...
3314 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3315 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3316 const auto *DestVar =
3317 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3318 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3319
3320 const auto *SrcVar =
3321 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3322 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3323
3324 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3325 QualType Type = VD->getType();
3326 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3327 }
3328 CGF.FinishFunction();
3329 return Fn;
3330}
3331
3332void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3333 const RegionCodeGenTy &SingleOpGen,
3334 SourceLocation Loc,
3335 ArrayRef<const Expr *> CopyprivateVars,
3336 ArrayRef<const Expr *> SrcExprs,
3337 ArrayRef<const Expr *> DstExprs,
3338 ArrayRef<const Expr *> AssignmentOps) {
3339 if (!CGF.HaveInsertPoint())
3340 return;
3341 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
3342 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
3343 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
;
3344 ASTContext &C = CGM.getContext();
3345 // int32 did_it = 0;
3346 // if(__kmpc_single(ident_t *, gtid)) {
3347 // SingleOpGen();
3348 // __kmpc_end_single(ident_t *, gtid);
3349 // did_it = 1;
3350 // }
3351 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3352 // <copy_func>, did_it);
3353
3354 Address DidIt = Address::invalid();
3355 if (!CopyprivateVars.empty()) {
3356 // int32 did_it = 0;
3357 QualType KmpInt32Ty =
3358 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3359 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3360 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3361 }
3362 // Prepare arguments and build a call to __kmpc_single
3363 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3364 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3365 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3366 /*Conditional=*/true);
3367 SingleOpGen.setAction(Action);
3368 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3369 if (DidIt.isValid()) {
3370 // did_it = 1;
3371 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3372 }
3373 Action.Done(CGF);
3374 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3375 // <copy_func>, did_it);
3376 if (DidIt.isValid()) {
3377 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3378 QualType CopyprivateArrayTy =
3379 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3380 /*IndexTypeQuals=*/0);
3381 // Create a list of all private variables for copyprivate.
3382 Address CopyprivateList =
3383 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3384 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3385 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3386 CGF.Builder.CreateStore(
3387 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3388 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3389 Elem);
3390 }
3391 // Build function that copies private values from single region to all other
3392 // threads in the corresponding parallel region.
3393 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3394 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3395 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3396 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3397 Address CL =
3398 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3399 CGF.VoidPtrTy);
3400 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3401 llvm::Value *Args[] = {
3402 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3403 getThreadID(CGF, Loc), // i32 <gtid>
3404 BufSize, // size_t <buf_size>
3405 CL.getPointer(), // void *<copyprivate list>
3406 CpyFn, // void (*) (void *, void *) <copy_func>
3407 DidItVal // i32 did_it
3408 };
3409 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3410 }
3411}
3412
3413void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3414 const RegionCodeGenTy &OrderedOpGen,
3415 SourceLocation Loc, bool IsThreads) {
3416 if (!CGF.HaveInsertPoint())
3417 return;
3418 // __kmpc_ordered(ident_t *, gtid);
3419 // OrderedOpGen();
3420 // __kmpc_end_ordered(ident_t *, gtid);
3421 // Prepare arguments and build a call to __kmpc_ordered
3422 if (IsThreads) {
3423 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3424 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3425 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3426 Args);
3427 OrderedOpGen.setAction(Action);
3428 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3429 return;
3430 }
3431 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3432}
3433
3434unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3435 unsigned Flags;
3436 if (Kind == OMPD_for)
3437 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3438 else if (Kind == OMPD_sections)
3439 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3440 else if (Kind == OMPD_single)
3441 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3442 else if (Kind == OMPD_barrier)
3443 Flags = OMP_IDENT_BARRIER_EXPL;
3444 else
3445 Flags = OMP_IDENT_BARRIER_IMPL;
3446 return Flags;
3447}
3448
3449void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3450 CodeGenFunction &CGF, const OMPLoopDirective &S,
3451 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3452 // Check if the loop directive is actually a doacross loop directive. In this
3453 // case choose static, 1 schedule.
3454 if (llvm::any_of(
3455 S.getClausesOfKind<OMPOrderedClause>(),
3456 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3457 ScheduleKind = OMPC_SCHEDULE_static;
3458 // Chunk size is 1 in this case.
3459 llvm::APInt ChunkSize(32, 1);
3460 ChunkExpr = IntegerLiteral::Create(
3461 CGF.getContext(), ChunkSize,
3462 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3463 SourceLocation());
3464 }
3465}
3466
3467void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3468 OpenMPDirectiveKind Kind, bool EmitChecks,
3469 bool ForceSimpleCall) {
3470 if (!CGF.HaveInsertPoint())
3471 return;
3472 // Build call __kmpc_cancel_barrier(loc, thread_id);
3473 // Build call __kmpc_barrier(loc, thread_id);
3474 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3475 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3476 // thread_id);
3477 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3478 getThreadID(CGF, Loc)};
3479 if (auto *OMPRegionInfo =
3480 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3481 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3482 llvm::Value *Result = CGF.EmitRuntimeCall(
3483 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3484 if (EmitChecks) {
3485 // if (__kmpc_cancel_barrier()) {
3486 // exit from construct;
3487 // }
3488 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3489 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3490 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3491 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3492 CGF.EmitBlock(ExitBB);
3493 // exit from construct;
3494 CodeGenFunction::JumpDest CancelDestination =
3495 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3496 CGF.EmitBranchThroughCleanup(CancelDestination);
3497 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3498 }
3499 return;
3500 }
3501 }
3502 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3503}
3504
3505/// Map the OpenMP loop schedule to the runtime enumeration.
3506static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3507 bool Chunked, bool Ordered) {
3508 switch (ScheduleKind) {
3509 case OMPC_SCHEDULE_static:
3510 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3511 : (Ordered ? OMP_ord_static : OMP_sch_static);
3512 case OMPC_SCHEDULE_dynamic:
3513 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3514 case OMPC_SCHEDULE_guided:
3515 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3516 case OMPC_SCHEDULE_runtime:
3517 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3518 case OMPC_SCHEDULE_auto:
3519 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3520 case OMPC_SCHEDULE_unknown:
3521 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3521, __PRETTY_FUNCTION__))
;
3522 return Ordered ? OMP_ord_static : OMP_sch_static;
3523 }
3524 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3524)
;
3525}
3526
3527/// Map the OpenMP distribute schedule to the runtime enumeration.
3528static OpenMPSchedType
3529getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3530 // only static is allowed for dist_schedule
3531 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3532}
3533
3534bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3535 bool Chunked) const {
3536 OpenMPSchedType Schedule =
3537 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3538 return Schedule == OMP_sch_static;
3539}
3540
3541bool CGOpenMPRuntime::isStaticNonchunked(
3542 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3543 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3544 return Schedule == OMP_dist_sch_static;
3545}
3546
3547bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3548 bool Chunked) const {
3549 OpenMPSchedType Schedule =
3550 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3551 return Schedule == OMP_sch_static_chunked;
3552}
3553
3554bool CGOpenMPRuntime::isStaticChunked(
3555 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3556 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3557 return Schedule == OMP_dist_sch_static_chunked;
3558}
3559
3560bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3561 OpenMPSchedType Schedule =
3562 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3563 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3563, __PRETTY_FUNCTION__))
;
3564 return Schedule != OMP_sch_static;
3565}
3566
3567static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3568 OpenMPScheduleClauseModifier M1,
3569 OpenMPScheduleClauseModifier M2) {
3570 int Modifier = 0;
3571 switch (M1) {
3572 case OMPC_SCHEDULE_MODIFIER_monotonic:
3573 Modifier = OMP_sch_modifier_monotonic;
3574 break;
3575 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3576 Modifier = OMP_sch_modifier_nonmonotonic;
3577 break;
3578 case OMPC_SCHEDULE_MODIFIER_simd:
3579 if (Schedule == OMP_sch_static_chunked)
3580 Schedule = OMP_sch_static_balanced_chunked;
3581 break;
3582 case OMPC_SCHEDULE_MODIFIER_last:
3583 case OMPC_SCHEDULE_MODIFIER_unknown:
3584 break;
3585 }
3586 switch (M2) {
3587 case OMPC_SCHEDULE_MODIFIER_monotonic:
3588 Modifier = OMP_sch_modifier_monotonic;
3589 break;
3590 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3591 Modifier = OMP_sch_modifier_nonmonotonic;
3592 break;
3593 case OMPC_SCHEDULE_MODIFIER_simd:
3594 if (Schedule == OMP_sch_static_chunked)
3595 Schedule = OMP_sch_static_balanced_chunked;
3596 break;
3597 case OMPC_SCHEDULE_MODIFIER_last:
3598 case OMPC_SCHEDULE_MODIFIER_unknown:
3599 break;
3600 }
3601 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3602 // If the static schedule kind is specified or if the ordered clause is
3603 // specified, and if the nonmonotonic modifier is not specified, the effect is
3604 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3605 // modifier is specified, the effect is as if the nonmonotonic modifier is
3606 // specified.
3607 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3608 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3609 Schedule == OMP_sch_static_balanced_chunked ||
3610 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static))
3611 Modifier = OMP_sch_modifier_nonmonotonic;
3612 }
3613 return Schedule | Modifier;
3614}
3615
3616void CGOpenMPRuntime::emitForDispatchInit(
3617 CodeGenFunction &CGF, SourceLocation Loc,
3618 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3619 bool Ordered, const DispatchRTInput &DispatchValues) {
3620 if (!CGF.HaveInsertPoint())
3621 return;
3622 OpenMPSchedType Schedule = getRuntimeSchedule(
3623 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3624 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3625 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3626 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3627 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
;
3628 // Call __kmpc_dispatch_init(
3629 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3630 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3631 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3632
3633 // If the Chunk was not specified in the clause - use default value 1.
3634 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3635 : CGF.Builder.getIntN(IVSize, 1);
3636 llvm::Value *Args[] = {
3637 emitUpdateLocation(CGF, Loc),
3638 getThreadID(CGF, Loc),
3639 CGF.Builder.getInt32(addMonoNonMonoModifier(
3640 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3641 DispatchValues.LB, // Lower
3642 DispatchValues.UB, // Upper
3643 CGF.Builder.getIntN(IVSize, 1), // Stride
3644 Chunk // Chunk
3645 };
3646 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3647}
3648
3649static void emitForStaticInitCall(
3650 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3651 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3652 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3653 const CGOpenMPRuntime::StaticRTInput &Values) {
3654 if (!CGF.HaveInsertPoint())
3655 return;
3656
3657 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3657, __PRETTY_FUNCTION__))
;
3658 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3659 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3660 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3661 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3662 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
;
3663
3664 // Call __kmpc_for_static_init(
3665 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3666 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3667 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3668 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3669 llvm::Value *Chunk = Values.Chunk;
3670 if (Chunk == nullptr) {
3671 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
3672 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
3673 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
;
3674 // If the Chunk was not specified in the clause - use default value 1.
3675 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3676 } else {
3677 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3678 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3679 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3680 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3681 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
;
3682 }
3683 llvm::Value *Args[] = {
3684 UpdateLocation,
3685 ThreadId,
3686 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3687 M2)), // Schedule type
3688 Values.IL.getPointer(), // &isLastIter
3689 Values.LB.getPointer(), // &LB
3690 Values.UB.getPointer(), // &UB
3691 Values.ST.getPointer(), // &Stride
3692 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3693 Chunk // Chunk
3694 };
3695 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3696}
3697
3698void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3699 SourceLocation Loc,
3700 OpenMPDirectiveKind DKind,
3701 const OpenMPScheduleTy &ScheduleKind,
3702 const StaticRTInput &Values) {
3703 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3704 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3705 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3706 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3708 isOpenMPLoopDirective(DKind)
3709 ? OMP_IDENT_WORK_LOOP
3710 : OMP_IDENT_WORK_SECTIONS);
3711 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3712 llvm::FunctionCallee StaticInitFunction =
3713 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3714 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3715 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3716}
3717
3718void CGOpenMPRuntime::emitDistributeStaticInit(
3719 CodeGenFunction &CGF, SourceLocation Loc,
3720 OpenMPDistScheduleClauseKind SchedKind,
3721 const CGOpenMPRuntime::StaticRTInput &Values) {
3722 OpenMPSchedType ScheduleNum =
3723 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3724 llvm::Value *UpdatedLocation =
3725 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3726 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3727 llvm::FunctionCallee StaticInitFunction =
3728 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3729 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3730 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3731 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3732}
3733
3734void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3735 SourceLocation Loc,
3736 OpenMPDirectiveKind DKind) {
3737 if (!CGF.HaveInsertPoint())
3738 return;
3739 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3740 llvm::Value *Args[] = {
3741 emitUpdateLocation(CGF, Loc,
3742 isOpenMPDistributeDirective(DKind)
3743 ? OMP_IDENT_WORK_DISTRIBUTE
3744 : isOpenMPLoopDirective(DKind)
3745 ? OMP_IDENT_WORK_LOOP
3746 : OMP_IDENT_WORK_SECTIONS),
3747 getThreadID(CGF, Loc)};
3748 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3749 Args);
3750}
3751
3752void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3753 SourceLocation Loc,
3754 unsigned IVSize,
3755 bool IVSigned) {
3756 if (!CGF.HaveInsertPoint())
3757 return;
3758 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3759 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3760 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3761}
3762
3763llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3764 SourceLocation Loc, unsigned IVSize,
3765 bool IVSigned, Address IL,
3766 Address LB, Address UB,
3767 Address ST) {
3768 // Call __kmpc_dispatch_next(
3769 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3770 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3771 // kmp_int[32|64] *p_stride);
3772 llvm::Value *Args[] = {
3773 emitUpdateLocation(CGF, Loc),
3774 getThreadID(CGF, Loc),
3775 IL.getPointer(), // &isLastIter
3776 LB.getPointer(), // &Lower
3777 UB.getPointer(), // &Upper
3778 ST.getPointer() // &Stride
3779 };
3780 llvm::Value *Call =
3781 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3782 return CGF.EmitScalarConversion(
3783 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3784 CGF.getContext().BoolTy, Loc);
3785}
3786
3787void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3788 llvm::Value *NumThreads,
3789 SourceLocation Loc) {
3790 if (!CGF.HaveInsertPoint())
3791 return;
3792 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3793 llvm::Value *Args[] = {
3794 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3795 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3796 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3797 Args);
3798}
3799
3800void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3801 OpenMPProcBindClauseKind ProcBind,
3802 SourceLocation Loc) {
3803 if (!CGF.HaveInsertPoint())
3804 return;
3805 // Constants for proc bind value accepted by the runtime.
3806 enum ProcBindTy {
3807 ProcBindFalse = 0,
3808 ProcBindTrue,
3809 ProcBindMaster,
3810 ProcBindClose,
3811 ProcBindSpread,
3812 ProcBindIntel,
3813 ProcBindDefault
3814 } RuntimeProcBind;
3815 switch (ProcBind) {
3816 case OMPC_PROC_BIND_master:
3817 RuntimeProcBind = ProcBindMaster;
3818 break;
3819 case OMPC_PROC_BIND_close:
3820 RuntimeProcBind = ProcBindClose;
3821 break;
3822 case OMPC_PROC_BIND_spread:
3823 RuntimeProcBind = ProcBindSpread;
3824 break;
3825 case OMPC_PROC_BIND_unknown:
3826 llvm_unreachable("Unsupported proc_bind value.")::llvm::llvm_unreachable_internal("Unsupported proc_bind value."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3826)
;
3827 }
3828 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3829 llvm::Value *Args[] = {
3830 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3831 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3832 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3833}
3834
3835void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3836 SourceLocation Loc) {
3837 if (!CGF.HaveInsertPoint())
3838 return;
3839 // Build call void __kmpc_flush(ident_t *loc)
3840 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3841 emitUpdateLocation(CGF, Loc));
3842}
3843
3844namespace {
3845/// Indexes of fields for type kmp_task_t.
3846enum KmpTaskTFields {
3847 /// List of shared variables.
3848 KmpTaskTShareds,
3849 /// Task routine.
3850 KmpTaskTRoutine,
3851 /// Partition id for the untied tasks.
3852 KmpTaskTPartId,
3853 /// Function with call of destructors for private variables.
3854 Data1,
3855 /// Task priority.
3856 Data2,
3857 /// (Taskloops only) Lower bound.
3858 KmpTaskTLowerBound,
3859 /// (Taskloops only) Upper bound.
3860 KmpTaskTUpperBound,
3861 /// (Taskloops only) Stride.
3862 KmpTaskTStride,
3863 /// (Taskloops only) Is last iteration flag.
3864 KmpTaskTLastIter,
3865 /// (Taskloops only) Reduction data.
3866 KmpTaskTReductions,
3867};
3868} // anonymous namespace
3869
3870bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3871 return OffloadEntriesTargetRegion.empty() &&
3872 OffloadEntriesDeviceGlobalVar.empty();
3873}
3874
3875/// Initialize target region entry.
3876void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3877 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3878 StringRef ParentName, unsigned LineNum,
3879 unsigned Order) {
3880 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
3881 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
3882 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
;
3883 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3884 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3885 OMPTargetRegionEntryTargetRegion);
3886 ++OffloadingEntriesNum;
3887}
3888
3889void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3890 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3891 StringRef ParentName, unsigned LineNum,
3892 llvm::Constant *Addr, llvm::Constant *ID,
3893 OMPTargetRegionEntryKind Flags) {
3894 // If we are emitting code for a target, the entry is already initialized,
3895 // only has to be registered.
3896 if (CGM.getLangOpts().OpenMPIsDevice) {
3897 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3898 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3899 DiagnosticsEngine::Error,
3900 "Unable to find target region on line '%0' in the device code.");
3901 CGM.getDiags().Report(DiagID) << LineNum;
3902 return;
3903 }
3904 auto &Entry =
3905 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3906 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3906, __PRETTY_FUNCTION__))
;
3907 Entry.setAddress(Addr);
3908 Entry.setID(ID);
3909 Entry.setFlags(Flags);
3910 } else {
3911 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3912 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3913 ++OffloadingEntriesNum;
3914 }
3915}
3916
3917bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3918 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3919 unsigned LineNum) const {
3920 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3921 if (PerDevice == OffloadEntriesTargetRegion.end())
3922 return false;
3923 auto PerFile = PerDevice->second.find(FileID);
3924 if (PerFile == PerDevice->second.end())
3925 return false;
3926 auto PerParentName = PerFile->second.find(ParentName);
3927 if (PerParentName == PerFile->second.end())
3928 return false;
3929 auto PerLine = PerParentName->second.find(LineNum);
3930 if (PerLine == PerParentName->second.end())
3931 return false;
3932 // Fail if this entry is already registered.
3933 if (PerLine->second.getAddress() || PerLine->second.getID())
3934 return false;
3935 return true;
3936}
3937
3938void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3939 const OffloadTargetRegionEntryInfoActTy &Action) {
3940 // Scan all target region entries and perform the provided action.
3941 for (const auto &D : OffloadEntriesTargetRegion)
3942 for (const auto &F : D.second)
3943 for (const auto &P : F.second)
3944 for (const auto &L : P.second)
3945 Action(D.first, F.first, P.first(), L.first, L.second);
3946}
3947
3948void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3949 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3950 OMPTargetGlobalVarEntryKind Flags,
3951 unsigned Order) {
3952 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
3953 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
3954 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
;
3955 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3956 ++OffloadingEntriesNum;
3957}
3958
3959void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3960 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3961 CharUnits VarSize,
3962 OMPTargetGlobalVarEntryKind Flags,
3963 llvm::GlobalValue::LinkageTypes Linkage) {
3964 if (CGM.getLangOpts().OpenMPIsDevice) {
3965 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3966 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3967, __PRETTY_FUNCTION__))
3967 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3967, __PRETTY_FUNCTION__))
;
3968 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
3969 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
;
3970 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3971 if (Entry.getVarSize().isZero()) {
3972 Entry.setVarSize(VarSize);
3973 Entry.setLinkage(Linkage);
3974 }
3975 return;
3976 }
3977 Entry.setVarSize(VarSize);
3978 Entry.setLinkage(Linkage);
3979 Entry.setAddress(Addr);
3980 } else {
3981 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3982 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3983 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3984, __PRETTY_FUNCTION__))
3984 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3984, __PRETTY_FUNCTION__))
;
3985 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
3986 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
;
3987 if (Entry.getVarSize().isZero()) {
3988 Entry.setVarSize(VarSize);
3989 Entry.setLinkage(Linkage);
3990 }
3991 return;
3992 }
3993 OffloadEntriesDeviceGlobalVar.try_emplace(
3994 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3995 ++OffloadingEntriesNum;
3996 }
3997}
3998
3999void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4000 actOnDeviceGlobalVarEntriesInfo(
4001 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4002 // Scan all target region entries and perform the provided action.
4003 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4004 Action(E.getKey(), E.getValue());
4005}
4006
4007llvm::Function *
4008CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
4009 // If we don't have entries or if we are emitting code for the device, we
4010 // don't need to do anything.
4011 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
4012 return nullptr;
4013
4014 llvm::Module &M = CGM.getModule();
4015 ASTContext &C = CGM.getContext();
4016
4017 // Get list of devices we care about
4018 const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
4019
4020 // We should be creating an offloading descriptor only if there are devices
4021 // specified.
4022 assert(!Devices.empty() && "No OpenMP offloading devices??")((!Devices.empty() && "No OpenMP offloading devices??"
) ? static_cast<void> (0) : __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4022, __PRETTY_FUNCTION__))
;
4023
4024 // Create the external variables that will point to the begin and end of the
4025 // host entries section. These will be defined by the linker.
4026 llvm::Type *OffloadEntryTy =
4027 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
4028 auto *HostEntriesBegin = new llvm::GlobalVariable(
4029 M, OffloadEntryTy, /*isConstant=*/true,
4030 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
4031 "__start_omp_offloading_entries");
4032 HostEntriesBegin->setVisibility(llvm::GlobalValue::HiddenVisibility);
4033 auto *HostEntriesEnd = new llvm::GlobalVariable(
4034 M, OffloadEntryTy, /*isConstant=*/true,
4035 llvm::GlobalValue::ExternalLinkage,
4036 /*Initializer=*/nullptr, "__stop_omp_offloading_entries");
4037 HostEntriesEnd->setVisibility(llvm::GlobalValue::HiddenVisibility);
4038
4039 // Create all device images
4040 auto *DeviceImageTy = cast<llvm::StructType>(
4041 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
4042 ConstantInitBuilder DeviceImagesBuilder(CGM);
4043 ConstantArrayBuilder DeviceImagesEntries =
4044 DeviceImagesBuilder.beginArray(DeviceImageTy);
4045
4046 for (const llvm::Triple &Device : Devices) {
4047 StringRef T = Device.getTriple();
4048 std::string BeginName = getName({"omp_offloading", "img_start", ""});
4049 auto *ImgBegin = new llvm::GlobalVariable(
4050 M, CGM.Int8Ty, /*isConstant=*/true,
4051 llvm::GlobalValue::ExternalWeakLinkage,
4052 /*Initializer=*/nullptr, Twine(BeginName).concat(T));
4053 std::string EndName = getName({"omp_offloading", "img_end", ""});
4054 auto *ImgEnd = new llvm::GlobalVariable(
4055 M, CGM.Int8Ty, /*isConstant=*/true,
4056 llvm::GlobalValue::ExternalWeakLinkage,
4057 /*Initializer=*/nullptr, Twine(EndName).concat(T));
4058
4059 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
4060 HostEntriesEnd};
4061 createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
4062 DeviceImagesEntries);
4063 }
4064
4065 // Create device images global array.
4066 std::string ImagesName = getName({"omp_offloading", "device_images"});
4067 llvm::GlobalVariable *DeviceImages =
4068 DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
4069 CGM.getPointerAlign(),
4070 /*isConstant=*/true);
4071 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4072
4073 // This is a Zero array to be used in the creation of the constant expressions
4074 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
4075 llvm::Constant::getNullValue(CGM.Int32Ty)};
4076
4077 // Create the target region descriptor.
4078 llvm::Constant *Data[] = {
4079 llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
4080 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
4081 DeviceImages, Index),
4082 HostEntriesBegin, HostEntriesEnd};
4083 std::string Descriptor = getName({"omp_offloading", "descriptor"});
4084 llvm::GlobalVariable *Desc = createGlobalStruct(
4085 CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
4086
4087 // Emit code to register or unregister the descriptor at execution
4088 // startup or closing, respectively.
4089
4090 llvm::Function *UnRegFn;
4091 {
4092 FunctionArgList Args;
4093 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
4094 Args.push_back(&DummyPtr);
4095
4096 CodeGenFunction CGF(CGM);
4097 // Disable debug info for global (de-)initializer because they are not part
4098 // of some particular construct.
4099 CGF.disableDebugInfo();
4100 const auto &FI =
4101 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4102 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4103 std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
4104 UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
4105 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
4106 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
4107 Desc);
4108 CGF.FinishFunction();
4109 }
4110 llvm::Function *RegFn;
4111 {
4112 CodeGenFunction CGF(CGM);
4113 // Disable debug info for global (de-)initializer because they are not part
4114 // of some particular construct.
4115 CGF.disableDebugInfo();
4116 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
4117 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4118
4119 // Encode offload target triples into the registration function name. It
4120 // will serve as a comdat key for the registration/unregistration code for
4121 // this particular combination of offloading targets.
4122 SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
4123 RegFnNameParts[0] = "omp_offloading";
4124 RegFnNameParts[1] = "descriptor_reg";
4125 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
4126 [](const llvm::Triple &T) -> const std::string& {
4127 return T.getTriple();
4128 });
4129 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
4130 std::string Descriptor = getName(RegFnNameParts);
4131 RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
4132 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
4133 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
4134 // Create a variable to drive the registration and unregistration of the
4135 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
4136 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
4137 SourceLocation(), nullptr, C.CharTy,
4138 ImplicitParamDecl::Other);
4139 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
4140 CGF.FinishFunction();
4141 }
4142 if (CGM.supportsCOMDAT()) {
4143 // It is sufficient to call registration function only once, so create a
4144 // COMDAT group for registration/unregistration functions and associated
4145 // data. That would reduce startup time and code size. Registration
4146 // function serves as a COMDAT group key.
4147 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
4148 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
4149 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
4150 RegFn->setComdat(ComdatKey);
4151 UnRegFn->setComdat(ComdatKey);
4152 DeviceImages->setComdat(ComdatKey);
4153 Desc->setComdat(ComdatKey);
4154 }
4155 return RegFn;
4156}
4157
4158void CGOpenMPRuntime::createOffloadEntry(
4159 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4160 llvm::GlobalValue::LinkageTypes Linkage) {
4161 StringRef Name = Addr->getName();
4162 llvm::Module &M = CGM.getModule();
4163 llvm::LLVMContext &C = M.getContext();
4164
4165 // Create constant string with the name.
4166 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4167
4168 std::string StringName = getName({"omp_offloading", "entry_name"});
4169 auto *Str = new llvm::GlobalVariable(
4170 M, StrPtrInit->getType(), /*isConstant=*/true,
4171 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4172 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4173
4174 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4175 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4176 llvm::ConstantInt::get(CGM.SizeTy, Size),
4177 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4178 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4179 std::string EntryName = getName({"omp_offloading", "entry", ""});
4180 llvm::GlobalVariable *Entry = createGlobalStruct(
4181 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4182 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4183
4184 // The entry has to be created in the section the linker expects it to be.
4185 Entry->setSection("omp_offloading_entries");
4186}
4187
4188void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4189 // Emit the offloading entries and metadata so that the device codegen side
4190 // can easily figure out what to emit. The produced metadata looks like
4191 // this:
4192 //
4193 // !omp_offload.info = !{!1, ...}
4194 //
4195 // Right now we only generate metadata for function that contain target
4196 // regions.
4197
4198 // If we do not have entries, we don't need to do anything.
4199 if (OffloadEntriesInfoManager.empty())
4200 return;
4201
4202 llvm::Module &M = CGM.getModule();
4203 llvm::LLVMContext &C = M.getContext();
4204 SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
4205 OrderedEntries(OffloadEntriesInfoManager.size());
4206 llvm::SmallVector<StringRef, 16> ParentFunctions(
4207 OffloadEntriesInfoManager.size());
4208
4209 // Auxiliary methods to create metadata values and strings.
4210 auto &&GetMDInt = [this](unsigned V) {
4211 return llvm::ConstantAsMetadata::get(
4212 llvm::ConstantInt::get(CGM.Int32Ty, V));
4213 };
4214
4215 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4216
4217 // Create the offloading info metadata node.
4218 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4219
4220 // Create function that emits metadata for each target region entry;
4221 auto &&TargetRegionMetadataEmitter =
4222 [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
4223 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4224 unsigned Line,
4225 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4226 // Generate metadata for target regions. Each entry of this metadata
4227 // contains:
4228 // - Entry 0 -> Kind of this type of metadata (0).
4229 // - Entry 1 -> Device ID of the file where the entry was identified.
4230 // - Entry 2 -> File ID of the file where the entry was identified.
4231 // - Entry 3 -> Mangled name of the function where the entry was
4232 // identified.
4233 // - Entry 4 -> Line in the file where the entry was identified.
4234 // - Entry 5 -> Order the entry was created.
4235 // The first element of the metadata node is the kind.
4236 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4237 GetMDInt(FileID), GetMDString(ParentName),
4238 GetMDInt(Line), GetMDInt(E.getOrder())};
4239
4240 // Save this entry in the right position of the ordered entries array.
4241 OrderedEntries[E.getOrder()] = &E;
4242 ParentFunctions[E.getOrder()] = ParentName;
4243
4244 // Add metadata to the named metadata node.
4245 MD->addOperand(llvm::MDNode::get(C, Ops));
4246 };
4247
4248 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4249 TargetRegionMetadataEmitter);
4250
4251 // Create function that emits metadata for each device global variable entry;
4252 auto &&DeviceGlobalVarMetadataEmitter =
4253 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4254 MD](StringRef MangledName,
4255 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4256 &E) {
4257 // Generate metadata for global variables. Each entry of this metadata
4258 // contains:
4259 // - Entry 0 -> Kind of this type of metadata (1).
4260 // - Entry 1 -> Mangled name of the variable.
4261 // - Entry 2 -> Declare target kind.
4262 // - Entry 3 -> Order the entry was created.
4263 // The first element of the metadata node is the kind.
4264 llvm::Metadata *Ops[] = {
4265 GetMDInt(E.getKind()), GetMDString(MangledName),
4266 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4267
4268 // Save this entry in the right position of the ordered entries array.
4269 OrderedEntries[E.getOrder()] = &E;
4270
4271 // Add metadata to the named metadata node.
4272 MD->addOperand(llvm::MDNode::get(C, Ops));
4273 };
4274
4275 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4276 DeviceGlobalVarMetadataEmitter);
4277
4278 for (const auto *E : OrderedEntries) {
4279 assert(E && "All ordered entries must exist!")((E && "All ordered entries must exist!") ? static_cast
<void> (0) : __assert_fail ("E && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4279, __PRETTY_FUNCTION__))
;
4280 if (const auto *CE =
4281 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4282 E)) {
4283 if (!CE->getID() || !CE->getAddress()) {
4284 // Do not blame the entry if the parent funtion is not emitted.
4285 StringRef FnName = ParentFunctions[CE->getOrder()];
4286 if (!CGM.GetGlobalValue(FnName))
4287 continue;
4288 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4289 DiagnosticsEngine::Error,
4290 "Offloading entry for target region is incorrect: either the "
4291 "address or the ID is invalid.");
4292 CGM.getDiags().Report(DiagID);
4293 continue;
4294 }
4295 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4296 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4297 } else if (const auto *CE =
4298 dyn_cast<OffloadEntriesInfoManagerTy::
4299 OffloadEntryInfoDeviceGlobalVar>(E)) {
4300 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4301 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4302 CE->getFlags());
4303 switch (Flags) {
4304 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4305 if (CGM.getLangOpts().OpenMPIsDevice &&
4306 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4307 continue;
4308 if (!CE->getAddress()) {
4309 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4310 DiagnosticsEngine::Error,
4311 "Offloading entry for declare target variable is incorrect: the "
4312 "address is invalid.");
4313 CGM.getDiags().Report(DiagID);
4314 continue;
4315 }
4316 // The vaiable has no definition - no need to add the entry.
4317 if (CE->getVarSize().isZero())
4318 continue;
4319 break;
4320 }
4321 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4322 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
4323 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
4324 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
;
4325 if (CGM.getLangOpts().OpenMPIsDevice)
4326 continue;
4327 if (!CE->getAddress()) {
4328 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4329 DiagnosticsEngine::Error,
4330 "Offloading entry for declare target variable is incorrect: the "
4331 "address is invalid.");
4332 CGM.getDiags().Report(DiagID);
4333 continue;
4334 }
4335 break;
4336 }
4337 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4338 CE->getVarSize().getQuantity(), Flags,
4339 CE->getLinkage());
4340 } else {
4341 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4341)
;
4342 }
4343 }
4344}
4345
4346/// Loads all the offload entries information from the host IR
4347/// metadata.
4348void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4349 // If we are in target mode, load the metadata from the host IR. This code has
4350 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4351
4352 if (!CGM.getLangOpts().OpenMPIsDevice)
4353 return;
4354
4355 if (CGM.getLangOpts().OMPHostIRFile.empty())
4356 return;
4357
4358 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4359 if (auto EC = Buf.getError()) {
4360 CGM.getDiags().Report(diag::err_cannot_open_file)
4361 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4362 return;
4363 }
4364
4365 llvm::LLVMContext C;
4366 auto ME = expectedToErrorOrAndEmitErrors(
4367 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4368
4369 if (auto EC = ME.getError()) {
4370 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4371 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4372 CGM.getDiags().Report(DiagID)
4373 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4374 return;
4375 }
4376
4377 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4378 if (!MD)
4379 return;
4380
4381 for (llvm::MDNode *MN : MD->operands()) {
4382 auto &&GetMDInt = [MN](unsigned Idx) {
4383 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4384 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4385 };
4386
4387 auto &&GetMDString = [MN](unsigned Idx) {
4388 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4389 return V->getString();
4390 };
4391
4392 switch (GetMDInt(0)) {
4393 default:
4394 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4394)
;
4395 break;
4396 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4397 OffloadingEntryInfoTargetRegion:
4398 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4399 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4400 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4401 /*Order=*/GetMDInt(5));
4402 break;
4403 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4404 OffloadingEntryInfoDeviceGlobalVar:
4405 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4406 /*MangledName=*/GetMDString(1),
4407 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4408 /*Flags=*/GetMDInt(2)),
4409 /*Order=*/GetMDInt(3));
4410 break;
4411 }
4412 }
4413}
4414
4415void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4416 if (!KmpRoutineEntryPtrTy) {
4417 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4418 ASTContext &C = CGM.getContext();
4419 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4420 FunctionProtoType::ExtProtoInfo EPI;
4421 KmpRoutineEntryPtrQTy = C.getPointerType(
4422 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4423 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4424 }
4425}
4426
4427QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4428 // Make sure the type of the entry is already created. This is the type we
4429 // have to create:
4430 // struct __tgt_offload_entry{
4431 // void *addr; // Pointer to the offload entry info.
4432 // // (function or global)
4433 // char *name; // Name of the function or global.
4434 // size_t size; // Size of the entry info (0 if it a function).
4435 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4436 // int32_t reserved; // Reserved, to use by the runtime library.
4437 // };
4438 if (TgtOffloadEntryQTy.isNull()) {
4439 ASTContext &C = CGM.getContext();
4440 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4441 RD->startDefinition();
4442 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4443 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4444 addFieldToRecordDecl(C, RD, C.getSizeType());
4445 addFieldToRecordDecl(
4446 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4447 addFieldToRecordDecl(
4448 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4449 RD->completeDefinition();
4450 RD->addAttr(PackedAttr::CreateImplicit(C));
4451 TgtOffloadEntryQTy = C.getRecordType(RD);
4452 }
4453 return TgtOffloadEntryQTy;
4454}
4455
4456QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4457 // These are the types we need to build:
4458 // struct __tgt_device_image{
4459 // void *ImageStart; // Pointer to the target code start.
4460 // void *ImageEnd; // Pointer to the target code end.
4461 // // We also add the host entries to the device image, as it may be useful
4462 // // for the target runtime to have access to that information.
4463 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
4464 // // the entries.
4465 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4466 // // entries (non inclusive).
4467 // };
4468 if (TgtDeviceImageQTy.isNull()) {
4469 ASTContext &C = CGM.getContext();
4470 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4471 RD->startDefinition();
4472 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4473 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4474 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4475 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4476 RD->completeDefinition();
4477 TgtDeviceImageQTy = C.getRecordType(RD);
4478 }
4479 return TgtDeviceImageQTy;
4480}
4481
4482QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4483 // struct __tgt_bin_desc{
4484 // int32_t NumDevices; // Number of devices supported.
4485 // __tgt_device_image *DeviceImages; // Arrays of device images
4486 // // (one per device).
4487 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
4488 // // entries.
4489 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4490 // // entries (non inclusive).
4491 // };
4492 if (TgtBinaryDescriptorQTy.isNull()) {
4493 ASTContext &C = CGM.getContext();
4494 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4495 RD->startDefinition();
4496 addFieldToRecordDecl(
4497 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4498 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4499 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4500 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4501 RD->completeDefinition();
4502 TgtBinaryDescriptorQTy = C.getRecordType(RD);
4503 }
4504 return TgtBinaryDescriptorQTy;
4505}
4506
4507namespace {
4508struct PrivateHelpersTy {
4509 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4510 const VarDecl *PrivateElemInit)
4511 : Original(Original), PrivateCopy(PrivateCopy),
4512 PrivateElemInit(PrivateElemInit) {}
4513 const VarDecl *Original;
4514 const VarDecl *PrivateCopy;
4515 const VarDecl *PrivateElemInit;
4516};
4517typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4518} // anonymous namespace
4519
4520static RecordDecl *
4521createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4522 if (!Privates.empty()) {
4523 ASTContext &C = CGM.getContext();
4524 // Build struct .kmp_privates_t. {
4525 // /* private vars */
4526 // };
4527 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4528 RD->startDefinition();
4529 for (const auto &Pair : Privates) {
4530 const VarDecl *VD = Pair.second.Original;
4531 QualType Type = VD->getType().getNonReferenceType();
4532 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4533 if (VD->hasAttrs()) {
4534 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4535 E(VD->getAttrs().end());
4536 I != E; ++I)
4537 FD->addAttr(*I);
4538 }
4539 }
4540 RD->completeDefinition();
4541 return RD;
4542 }
4543 return nullptr;
4544}
4545
4546static RecordDecl *
4547createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4548 QualType KmpInt32Ty,
4549 QualType KmpRoutineEntryPointerQTy) {
4550 ASTContext &C = CGM.getContext();
4551 // Build struct kmp_task_t {
4552 // void * shareds;
4553 // kmp_routine_entry_t routine;
4554 // kmp_int32 part_id;
4555 // kmp_cmplrdata_t data1;
4556 // kmp_cmplrdata_t data2;
4557 // For taskloops additional fields:
4558 // kmp_uint64 lb;
4559 // kmp_uint64 ub;
4560 // kmp_int64 st;
4561 // kmp_int32 liter;
4562 // void * reductions;
4563 // };
4564 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4565 UD->startDefinition();
4566 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4567 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4568 UD->completeDefinition();
4569 QualType KmpCmplrdataTy = C.getRecordType(UD);
4570 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4571 RD->startDefinition();
4572 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4573 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4574 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4575 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4576 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4577 if (isOpenMPTaskLoopDirective(Kind)) {
4578 QualType KmpUInt64Ty =
4579 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4580 QualType KmpInt64Ty =
4581 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4582 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4583 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4584 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4585 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4586 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4587 }
4588 RD->completeDefinition();
4589 return RD;
4590}
4591
4592static RecordDecl *
4593createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4594 ArrayRef<PrivateDataTy> Privates) {
4595 ASTContext &C = CGM.getContext();
4596 // Build struct kmp_task_t_with_privates {
4597 // kmp_task_t task_data;
4598 // .kmp_privates_t. privates;
4599 // };
4600 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4601 RD->startDefinition();
4602 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4603 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4604 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4605 RD->completeDefinition();
4606 return RD;
4607}
4608
4609/// Emit a proxy function which accepts kmp_task_t as the second
4610/// argument.
4611/// \code
4612/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4613/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4614/// For taskloops:
4615/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4616/// tt->reductions, tt->shareds);
4617/// return 0;
4618/// }
4619/// \endcode
4620static llvm::Function *
4621emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4622 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4623 QualType KmpTaskTWithPrivatesPtrQTy,
4624 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4625 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4626 llvm::Value *TaskPrivatesMap) {
4627 ASTContext &C = CGM.getContext();
4628 FunctionArgList Args;
4629 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4630 ImplicitParamDecl::Other);
4631 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4632 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4633 ImplicitParamDecl::Other);
4634 Args.push_back(&GtidArg);
4635 Args.push_back(&TaskTypeArg);
4636 const auto &TaskEntryFnInfo =
4637 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4638 llvm::FunctionType *TaskEntryTy =
4639 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4640 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4641 auto *TaskEntry = llvm::Function::Create(
4642 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4643 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4644 TaskEntry->setDoesNotRecurse();
4645 CodeGenFunction CGF(CGM);
4646 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4647 Loc, Loc);
4648
4649 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4650 // tt,
4651 // For taskloops:
4652 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4653 // tt->task_data.shareds);
4654 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4655 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4656 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4657 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4658 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4659 const auto *KmpTaskTWithPrivatesQTyRD =
4660 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4661 LValue Base =
4662 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4663 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4664 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4665 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4666 llvm::Value *PartidParam = PartIdLVal.getPointer();
4667
4668 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4669 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4670 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4671 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4672 CGF.ConvertTypeForMem(SharedsPtrTy));
4673
4674 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4675 llvm::Value *PrivatesParam;
4676 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4677 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4678 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4679 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4680 } else {
4681 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4682 }
4683
4684 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4685 TaskPrivatesMap,
4686 CGF.Builder
4687 .CreatePointerBitCastOrAddrSpaceCast(
4688 TDBase.getAddress(), CGF.VoidPtrTy)
4689 .getPointer()};
4690 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4691 std::end(CommonArgs));
4692 if (isOpenMPTaskLoopDirective(Kind)) {
4693 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4694 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4695 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4696 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4697 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4698 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4699 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4700 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4701 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4702 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4703 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4704 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4705 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4706 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4707 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4708 CallArgs.push_back(LBParam);
4709 CallArgs.push_back(UBParam);
4710 CallArgs.push_back(StParam);
4711 CallArgs.push_back(LIParam);
4712 CallArgs.push_back(RParam);
4713 }
4714 CallArgs.push_back(SharedsParam);
4715
4716 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4717 CallArgs);
4718 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4719 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4720 CGF.FinishFunction();
4721 return TaskEntry;
4722}
4723
4724static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4725 SourceLocation Loc,
4726 QualType KmpInt32Ty,
4727 QualType KmpTaskTWithPrivatesPtrQTy,
4728 QualType KmpTaskTWithPrivatesQTy) {
4729 ASTContext &C = CGM.getContext();
4730 FunctionArgList Args;
4731 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4732 ImplicitParamDecl::Other);
4733 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4734 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4735 ImplicitParamDecl::Other);
4736 Args.push_back(&GtidArg);
4737 Args.push_back(&TaskTypeArg);
4738 const auto &DestructorFnInfo =
4739 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4740 llvm::FunctionType *DestructorFnTy =
4741 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4742 std::string Name =
4743 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4744 auto *DestructorFn =
4745 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4746 Name, &CGM.getModule());
4747 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4748 DestructorFnInfo);
4749 DestructorFn->setDoesNotRecurse();
4750 CodeGenFunction CGF(CGM);
4751 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4752 Args, Loc, Loc);
4753
4754 LValue Base = CGF.EmitLoadOfPointerLValue(
4755 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4756 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4757 const auto *KmpTaskTWithPrivatesQTyRD =
4758 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4759 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4760 Base = CGF.EmitLValueForField(Base, *FI);
4761 for (const auto *Field :
4762 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4763 if (QualType::DestructionKind DtorKind =
4764 Field->getType().isDestructedType()) {
4765 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4766 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4767 }
4768 }
4769 CGF.FinishFunction();
4770 return DestructorFn;
4771}
4772
4773/// Emit a privates mapping function for correct handling of private and
4774/// firstprivate variables.
4775/// \code
4776/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4777/// **noalias priv1,..., <tyn> **noalias privn) {
4778/// *priv1 = &.privates.priv1;
4779/// ...;
4780/// *privn = &.privates.privn;
4781/// }
4782/// \endcode
4783static llvm::Value *
4784emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4785 ArrayRef<const Expr *> PrivateVars,
4786 ArrayRef<const Expr *> FirstprivateVars,
4787 ArrayRef<const Expr *> LastprivateVars,
4788 QualType PrivatesQTy,
4789 ArrayRef<PrivateDataTy> Privates) {
4790 ASTContext &C = CGM.getContext();
4791 FunctionArgList Args;
4792 ImplicitParamDecl TaskPrivatesArg(
4793 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4794 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4795 ImplicitParamDecl::Other);
4796 Args.push_back(&TaskPrivatesArg);
4797 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4798 unsigned Counter = 1;
4799 for (const Expr *E : PrivateVars) {
4800 Args.push_back(ImplicitParamDecl::Create(
4801 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4802 C.getPointerType(C.getPointerType(E->getType()))
4803 .withConst()
4804 .withRestrict(),
4805 ImplicitParamDecl::Other));
4806 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4807 PrivateVarsPos[VD] = Counter;
4808 ++Counter;
4809 }
4810 for (const Expr *E : FirstprivateVars) {
4811 Args.push_back(ImplicitParamDecl::Create(
4812 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4813 C.getPointerType(C.getPointerType(E->getType()))
4814 .withConst()
4815 .withRestrict(),
4816 ImplicitParamDecl::Other));
4817 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4818 PrivateVarsPos[VD] = Counter;
4819 ++Counter;
4820 }
4821 for (const Expr *E : LastprivateVars) {
4822 Args.push_back(ImplicitParamDecl::Create(
4823 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4824 C.getPointerType(C.getPointerType(E->getType()))
4825 .withConst()
4826 .withRestrict(),
4827 ImplicitParamDecl::Other));
4828 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4829 PrivateVarsPos[VD] = Counter;
4830 ++Counter;
4831 }
4832 const auto &TaskPrivatesMapFnInfo =
4833 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4834 llvm::FunctionType *TaskPrivatesMapTy =
4835 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4836 std::string Name =
4837 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4838 auto *TaskPrivatesMap = llvm::Function::Create(
4839 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4840 &CGM.getModule());
4841 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4842 TaskPrivatesMapFnInfo);
4843 if (CGM.getLangOpts().Optimize) {
4844 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4845 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4846 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4847 }
4848 CodeGenFunction CGF(CGM);
4849 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4850 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4851
4852 // *privi = &.privates.privi;
4853 LValue Base = CGF.EmitLoadOfPointerLValue(
4854 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4855 TaskPrivatesArg.getType()->castAs<PointerType>());
4856 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4857 Counter = 0;
4858 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4859 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4860 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4861 LValue RefLVal =
4862 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4863 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4864 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4865 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4866 ++Counter;
4867 }
4868 CGF.FinishFunction();
4869 return TaskPrivatesMap;
4870}
4871
4872/// Emit initialization for private variables in task-based directives.
4873static void emitPrivatesInit(CodeGenFunction &CGF,
4874 const OMPExecutableDirective &D,
4875 Address KmpTaskSharedsPtr, LValue TDBase,
4876 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4877 QualType SharedsTy, QualType SharedsPtrTy,
4878 const OMPTaskDataTy &Data,
4879 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4880 ASTContext &C = CGF.getContext();
4881 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4882 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4883 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4884 ? OMPD_taskloop
4885 : OMPD_task;
4886 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4887 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4888 LValue SrcBase;
4889 bool IsTargetTask =
4890 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4891 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4892 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4893 // PointersArray and SizesArray. The original variables for these arrays are
4894 // not captured and we get their addresses explicitly.
4895 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4896 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4897 SrcBase = CGF.MakeAddrLValue(
4898 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4899 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4900 SharedsTy);
4901 }
4902 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4903 for (const PrivateDataTy &Pair : Privates) {
4904 const VarDecl *VD = Pair.second.PrivateCopy;
4905 const Expr *Init = VD->getAnyInitializer();
4906 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4907 !CGF.isTrivialInitializer(Init)))) {
4908 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4909 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4910 const VarDecl *OriginalVD = Pair.second.Original;
4911 // Check if the variable is the target-based BasePointersArray,
4912 // PointersArray or SizesArray.
4913 LValue SharedRefLValue;
4914 QualType Type = PrivateLValue.getType();
4915 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4916 if (IsTargetTask && !SharedField) {
4917 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4918 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4919 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4920 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4921 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4922 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4923 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4924 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
;
4925 SharedRefLValue =
4926 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4927 } else {
4928 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4929 SharedRefLValue = CGF.MakeAddrLValue(
4930 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4931 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4932 SharedRefLValue.getTBAAInfo());
4933 }
4934 if (Type->isArrayType()) {
4935 // Initialize firstprivate array.
4936 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4937 // Perform simple memcpy.
4938 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4939 } else {
4940 // Initialize firstprivate array using element-by-element
4941 // initialization.
4942 CGF.EmitOMPAggregateAssign(
4943 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4944 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4945 Address SrcElement) {
4946 // Clean up any temporaries needed by the initialization.
4947 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4948 InitScope.addPrivate(
4949 Elem, [SrcElement]() -> Address { return SrcElement; });
4950 (void)InitScope.Privatize();
4951 // Emit initialization for single element.
4952 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4953 CGF, &CapturesInfo);
4954 CGF.EmitAnyExprToMem(Init, DestElement,
4955 Init->getType().getQualifiers(),
4956 /*IsInitializer=*/false);
4957 });
4958 }
4959 } else {
4960 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4961 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4962 return SharedRefLValue.getAddress();
4963 });
4964 (void)InitScope.Privatize();
4965 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4966 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4967 /*capturedByInit=*/false);
4968 }
4969 } else {
4970 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4971 }
4972 }
4973 ++FI;
4974 }
4975}
4976
4977/// Check if duplication function is required for taskloops.
4978static bool checkInitIsRequired(CodeGenFunction &CGF,
4979 ArrayRef<PrivateDataTy> Privates) {
4980 bool InitRequired = false;
4981 for (const PrivateDataTy &Pair : Privates) {
4982 const VarDecl *VD = Pair.second.PrivateCopy;
4983 const Expr *Init = VD->getAnyInitializer();
4984 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4985 !CGF.isTrivialInitializer(Init));
4986 if (InitRequired)
4987 break;
4988 }
4989 return InitRequired;
4990}
4991
4992
4993/// Emit task_dup function (for initialization of
4994/// private/firstprivate/lastprivate vars and last_iter flag)
4995/// \code
4996/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4997/// lastpriv) {
4998/// // setup lastprivate flag
4999/// task_dst->last = lastpriv;
5000/// // could be constructor calls here...
5001/// }
5002/// \endcode
5003static llvm::Value *
5004emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
5005 const OMPExecutableDirective &D,
5006 QualType KmpTaskTWithPrivatesPtrQTy,
5007 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
5008 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
5009 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
5010 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
5011 ASTContext &C = CGM.getContext();
5012 FunctionArgList Args;
5013 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5014 KmpTaskTWithPrivatesPtrQTy,
5015 ImplicitParamDecl::Other);
5016 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5017 KmpTaskTWithPrivatesPtrQTy,
5018 ImplicitParamDecl::Other);
5019 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
5020 ImplicitParamDecl::Other);
5021 Args.push_back(&DstArg);
5022 Args.push_back(&SrcArg);
5023 Args.push_back(&LastprivArg);
5024 const auto &TaskDupFnInfo =
5025 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5026 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
5027 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
5028 auto *TaskDup = llvm::Function::Create(
5029 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
5030 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
5031 TaskDup->setDoesNotRecurse();
5032 CodeGenFunction CGF(CGM);
5033 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
5034 Loc);
5035
5036 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5037 CGF.GetAddrOfLocalVar(&DstArg),
5038 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5039 // task_dst->liter = lastpriv;
5040 if (WithLastIter) {
5041 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
5042 LValue Base = CGF.EmitLValueForField(
5043 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5044 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
5045 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
5046 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
5047 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
5048 }
5049
5050 // Emit initial values for private copies (if any).
5051 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5051, __PRETTY_FUNCTION__))
;
5052 Address KmpTaskSharedsPtr = Address::invalid();
5053 if (!Data.FirstprivateVars.empty()) {
5054 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5055 CGF.GetAddrOfLocalVar(&SrcArg),
5056 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5057 LValue Base = CGF.EmitLValueForField(
5058 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5059 KmpTaskSharedsPtr = Address(
5060 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
5061 Base, *std::next(KmpTaskTQTyRD->field_begin(),
5062 KmpTaskTShareds)),
5063 Loc),
5064 CGF.getNaturalTypeAlignment(SharedsTy));
5065 }
5066 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
5067 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
5068 CGF.FinishFunction();
5069 return TaskDup;
5070}
5071
5072/// Checks if destructor function is required to be generated.
5073/// \return true if cleanups are required, false otherwise.
5074static bool
5075checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
5076 bool NeedsCleanup = false;
5077 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
5078 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
5079 for (const FieldDecl *FD : PrivateRD->fields()) {
5080 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
5081 if (NeedsCleanup)
5082 break;
5083 }
5084 return NeedsCleanup;
5085}
5086
5087CGOpenMPRuntime::TaskResultTy
5088CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
5089 const OMPExecutableDirective &D,
5090 llvm::Function *TaskFunction, QualType SharedsTy,
5091 Address Shareds, const OMPTaskDataTy &Data) {
5092 ASTContext &C = CGM.getContext();
5093 llvm::SmallVector<PrivateDataTy, 4> Privates;
5094 // Aggregate privates and sort them by the alignment.
5095 auto I = Data.PrivateCopies.begin();
5096 for (const Expr *E : Data.PrivateVars) {
5097 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5098 Privates.emplace_back(
5099 C.getDeclAlign(VD),
5100 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5101 /*PrivateElemInit=*/nullptr));
5102 ++I;
5103 }
5104 I = Data.FirstprivateCopies.begin();
5105 auto IElemInitRef = Data.FirstprivateInits.begin();
5106 for (const Expr *E : Data.FirstprivateVars) {
5107 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5108 Privates.emplace_back(
5109 C.getDeclAlign(VD),
5110 PrivateHelpersTy(
5111 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5112 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
5113 ++I;
5114 ++IElemInitRef;
5115 }
5116 I = Data.LastprivateCopies.begin();
5117 for (const Expr *E : Data.LastprivateVars) {
5118 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5119 Privates.emplace_back(
5120 C.getDeclAlign(VD),
5121 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5122 /*PrivateElemInit=*/nullptr));
5123 ++I;
5124 }
5125 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5126 return L.first > R.first;
5127 });
5128 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5129 // Build type kmp_routine_entry_t (if not built yet).
5130 emitKmpRoutineEntryT(KmpInt32Ty);
5131 // Build type kmp_task_t (if not built yet).
5132 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5133 if (SavedKmpTaskloopTQTy.isNull()) {
5134 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5135 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5136 }
5137 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5138 } else {
5139 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5140 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5141 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5142 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
;
5143 if (SavedKmpTaskTQTy.isNull()) {
5144 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5145 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5146 }
5147 KmpTaskTQTy = SavedKmpTaskTQTy;
5148 }
5149 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5150 // Build particular struct kmp_task_t for the given task.
5151 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5152 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5153 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5154 QualType KmpTaskTWithPrivatesPtrQTy =
5155 C.getPointerType(KmpTaskTWithPrivatesQTy);
5156 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5157 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5158 KmpTaskTWithPrivatesTy->getPointerTo();
5159 llvm::Value *KmpTaskTWithPrivatesTySize =
5160 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5161 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5162
5163 // Emit initial values for private copies (if any).
5164 llvm::Value *TaskPrivatesMap = nullptr;
5165 llvm::Type *TaskPrivatesMapTy =
5166 std::next(TaskFunction->arg_begin(), 3)->getType();
5167 if (!Privates.empty()) {
5168 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5169 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5170 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5171 FI->getType(), Privates);
5172 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5173 TaskPrivatesMap, TaskPrivatesMapTy);
5174 } else {
5175 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5176 cast<llvm::PointerType>(TaskPrivatesMapTy));
5177 }
5178 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5179 // kmp_task_t *tt);
5180 llvm::Function *TaskEntry = emitProxyTaskFunction(
5181 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5182 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5183 TaskPrivatesMap);
5184
5185 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5186 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5187 // kmp_routine_entry_t *task_entry);
5188 // Task flags. Format is taken from
5189 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5190 // description of kmp_tasking_flags struct.
5191 enum {
5192 TiedFlag = 0x1,
5193 FinalFlag = 0x2,
5194 DestructorsFlag = 0x8,
5195 PriorityFlag = 0x20
5196 };
5197 unsigned Flags = Data.Tied ? TiedFlag : 0;
5198 bool NeedsCleanup = false;
5199 if (!Privates.empty()) {
5200 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5201 if (NeedsCleanup)
5202 Flags = Flags | DestructorsFlag;
5203 }
5204 if (Data.Priority.getInt())
5205 Flags = Flags | PriorityFlag;
5206 llvm::Value *TaskFlags =
5207 Data.Final.getPointer()
5208 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5209 CGF.Builder.getInt32(FinalFlag),
5210 CGF.Builder.getInt32(/*C=*/0))
5211 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5212 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5213 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5214 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5215 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5216 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5217 TaskEntry, KmpRoutineEntryPtrTy)};
5218 llvm::Value *NewTask;
5219 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5220 // Check if we have any device clause associated with the directive.
5221 const Expr *Device = nullptr;
5222 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5223 Device = C->getDevice();
5224 // Emit device ID if any otherwise use default value.
5225 llvm::Value *DeviceID;
5226 if (Device)
5227 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5228 CGF.Int64Ty, /*isSigned=*/true);
5229 else
5230 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5231 AllocArgs.push_back(DeviceID);
5232 NewTask = CGF.EmitRuntimeCall(
5233 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5234 } else {
5235 NewTask = CGF.EmitRuntimeCall(
5236 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5237 }
5238 llvm::Value *NewTaskNewTaskTTy =
5239 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5240 NewTask, KmpTaskTWithPrivatesPtrTy);
5241 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5242 KmpTaskTWithPrivatesQTy);
5243 LValue TDBase =
5244 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5245 // Fill the data in the resulting kmp_task_t record.
5246 // Copy shareds if there are any.
5247 Address KmpTaskSharedsPtr = Address::invalid();
5248 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5249 KmpTaskSharedsPtr =
5250 Address(CGF.EmitLoadOfScalar(
5251 CGF.EmitLValueForField(
5252 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5253 KmpTaskTShareds)),
5254 Loc),
5255 CGF.getNaturalTypeAlignment(SharedsTy));
5256 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5257 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5258 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5259 }
5260 // Emit initial values for private copies (if any).
5261 TaskResultTy Result;
5262 if (!Privates.empty()) {
5263 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5264 SharedsTy, SharedsPtrTy, Data, Privates,
5265 /*ForDup=*/false);
5266 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5267 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5268 Result.TaskDupFn = emitTaskDupFunction(
5269 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5270 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5271 /*WithLastIter=*/!Data.LastprivateVars.empty());
5272 }
5273 }
5274 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5275 enum { Priority = 0, Destructors = 1 };
5276 // Provide pointer to function with destructors for privates.
5277 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5278 const RecordDecl *KmpCmplrdataUD =
5279 (*FI)->getType()->getAsUnionType()->getDecl();
5280 if (NeedsCleanup) {
5281 llvm::Value *DestructorFn = emitDestructorsFunction(
5282 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5283 KmpTaskTWithPrivatesQTy);
5284 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5285 LValue DestructorsLV = CGF.EmitLValueForField(
5286 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5287 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5288 DestructorFn, KmpRoutineEntryPtrTy),
5289 DestructorsLV);
5290 }
5291 // Set priority.
5292 if (Data.Priority.getInt()) {
5293 LValue Data2LV = CGF.EmitLValueForField(
5294 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5295 LValue PriorityLV = CGF.EmitLValueForField(
5296 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5297 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5298 }
5299 Result.NewTask = NewTask;
5300 Result.TaskEntry = TaskEntry;
5301 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5302 Result.TDBase = TDBase;
5303 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5304 return Result;
5305}
5306
5307void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5308 const OMPExecutableDirective &D,
5309 llvm::Function *TaskFunction,
5310 QualType SharedsTy, Address Shareds,
5311 const Expr *IfCond,
5312 const OMPTaskDataTy &Data) {
5313 if (!CGF.HaveInsertPoint())
5314 return;
5315
5316 TaskResultTy Result =
5317 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5318 llvm::Value *NewTask = Result.NewTask;
5319 llvm::Function *TaskEntry = Result.TaskEntry;
5320 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5321 LValue TDBase = Result.TDBase;
5322 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5323 ASTContext &C = CGM.getContext();
5324 // Process list of dependences.
5325 Address DependenciesArray = Address::invalid();
5326 unsigned NumDependencies = Data.Dependences.size();
5327 if (NumDependencies) {
5328 // Dependence kind for RTL.
5329 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5330 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5331 RecordDecl *KmpDependInfoRD;
5332 QualType FlagsTy =
5333 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5334 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5335 if (KmpDependInfoTy.isNull()) {
5336 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5337 KmpDependInfoRD->startDefinition();
5338 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5339 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5340 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5341 KmpDependInfoRD->completeDefinition();
5342 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5343 } else {
5344 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5345 }
5346 // Define type kmp_depend_info[<Dependences.size()>];
5347 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5348 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5349 ArrayType::Normal, /*IndexTypeQuals=*/0);
5350 // kmp_depend_info[<Dependences.size()>] deps;
5351 DependenciesArray =
5352 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5353 for (unsigned I = 0; I < NumDependencies; ++I) {
5354 const Expr *E = Data.Dependences[I].second;
5355 LValue Addr = CGF.EmitLValue(E);
5356 llvm::Value *Size;
5357 QualType Ty = E->getType();
5358 if (const auto *ASE =
5359 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5360 LValue UpAddrLVal =
5361 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5362 llvm::Value *UpAddr =
5363 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5364 llvm::Value *LowIntPtr =
5365 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5366 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5367 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5368 } else {
5369 Size = CGF.getTypeSize(Ty);
5370 }
5371 LValue Base = CGF.MakeAddrLValue(
5372 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5373 KmpDependInfoTy);
5374 // deps[i].base_addr = &<Dependences[i].second>;
5375 LValue BaseAddrLVal = CGF.EmitLValueForField(
5376 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5377 CGF.EmitStoreOfScalar(
5378 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5379 BaseAddrLVal);
5380 // deps[i].len = sizeof(<Dependences[i].second>);
5381 LValue LenLVal = CGF.EmitLValueForField(
5382 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5383 CGF.EmitStoreOfScalar(Size, LenLVal);
5384 // deps[i].flags = <Dependences[i].first>;
5385 RTLDependenceKindTy DepKind;
5386 switch (Data.Dependences[I].first) {
5387 case OMPC_DEPEND_in:
5388 DepKind = DepIn;
5389 break;
5390 // Out and InOut dependencies must use the same code.
5391 case OMPC_DEPEND_out:
5392 case OMPC_DEPEND_inout:
5393 DepKind = DepInOut;
5394 break;
5395 case OMPC_DEPEND_mutexinoutset:
5396 DepKind = DepMutexInOutSet;
5397 break;
5398 case OMPC_DEPEND_source:
5399 case OMPC_DEPEND_sink:
5400 case OMPC_DEPEND_unknown:
5401 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5401)
;
5402 }
5403 LValue FlagsLVal = CGF.EmitLValueForField(
5404 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5405 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5406 FlagsLVal);
5407 }
5408 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5409 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5410 }
5411
5412 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5413 // libcall.
5414 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5415 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5416 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5417 // list is not empty
5418 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5419 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5420 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5421 llvm::Value *DepTaskArgs[7];
5422 if (NumDependencies) {
5423 DepTaskArgs[0] = UpLoc;
5424 DepTaskArgs[1] = ThreadID;
5425 DepTaskArgs[2] = NewTask;
5426 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5427 DepTaskArgs[4] = DependenciesArray.getPointer();
5428 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5429 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5430 }
5431 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5432 &TaskArgs,
5433 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5434 if (!Data.Tied) {
5435 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5436 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5437 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5438 }
5439 if (NumDependencies) {
5440 CGF.EmitRuntimeCall(
5441 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5442 } else {
5443 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5444 TaskArgs);
5445 }
5446 // Check if parent region is untied and build return for untied task;
5447 if (auto *Region =
5448 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5449 Region->emitUntiedSwitch(CGF);
5450 };
5451
5452 llvm::Value *DepWaitTaskArgs[6];
5453 if (NumDependencies) {
5454 DepWaitTaskArgs[0] = UpLoc;
5455 DepWaitTaskArgs[1] = ThreadID;
5456 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5457 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5458 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5459 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5460 }
5461 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5462 NumDependencies, &DepWaitTaskArgs,
5463 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5464 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5465 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5466 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5467 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5468 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5469 // is specified.
5470 if (NumDependencies)
5471 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5472 DepWaitTaskArgs);
5473 // Call proxy_task_entry(gtid, new_task);
5474 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5475 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5476 Action.Enter(CGF);
5477 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5478 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5479 OutlinedFnArgs);
5480 };
5481
5482 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5483 // kmp_task_t *new_task);
5484 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5485 // kmp_task_t *new_task);
5486 RegionCodeGenTy RCG(CodeGen);
5487 CommonActionTy Action(
5488 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5489 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5490 RCG.setAction(Action);
5491 RCG(CGF);
5492 };
5493
5494 if (IfCond) {
5495 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5496 } else {
5497 RegionCodeGenTy ThenRCG(ThenCodeGen);
5498 ThenRCG(CGF);
5499 }
5500}
5501
5502void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5503 const OMPLoopDirective &D,
5504 llvm::Function *TaskFunction,
5505 QualType SharedsTy, Address Shareds,
5506 const Expr *IfCond,
5507 const OMPTaskDataTy &Data) {
5508 if (!CGF.HaveInsertPoint())
5509 return;
5510 TaskResultTy Result =
5511 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5512 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5513 // libcall.
5514 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5515 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5516 // sched, kmp_uint64 grainsize, void *task_dup);
5517 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5518 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5519 llvm::Value *IfVal;
5520 if (IfCond) {
5521 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5522 /*isSigned=*/true);
5523 } else {
5524 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5525 }
5526
5527 LValue LBLVal = CGF.EmitLValueForField(
5528 Result.TDBase,
5529 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5530 const auto *LBVar =
5531 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5532 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5533 /*IsInitializer=*/true);
5534 LValue UBLVal = CGF.EmitLValueForField(
5535 Result.TDBase,
5536 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5537 const auto *UBVar =
5538 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5539 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5540 /*IsInitializer=*/true);
5541 LValue StLVal = CGF.EmitLValueForField(
5542 Result.TDBase,
5543 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5544 const auto *StVar =
5545 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5546 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5547 /*IsInitializer=*/true);
5548 // Store reductions address.
5549 LValue RedLVal = CGF.EmitLValueForField(
5550 Result.TDBase,
5551 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5552 if (Data.Reductions) {
5553 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5554 } else {
5555 CGF.EmitNullInitialization(RedLVal.getAddress(),
5556 CGF.getContext().VoidPtrTy);
5557 }
5558 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5559 llvm::Value *TaskArgs[] = {
5560 UpLoc,
5561 ThreadID,
5562 Result.NewTask,
5563 IfVal,
5564 LBLVal.getPointer(),
5565 UBLVal.getPointer(),
5566 CGF.EmitLoadOfScalar(StLVal, Loc),
5567 llvm::ConstantInt::getSigned(
5568 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5569 llvm::ConstantInt::getSigned(
5570 CGF.IntTy, Data.Schedule.getPointer()
5571 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5572 : NoSchedule),
5573 Data.Schedule.getPointer()
5574 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5575 /*isSigned=*/false)
5576 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5577 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5578 Result.TaskDupFn, CGF.VoidPtrTy)
5579 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5580 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5581}
5582
5583/// Emit reduction operation for each element of array (required for
5584/// array sections) LHS op = RHS.
5585/// \param Type Type of array.
5586/// \param LHSVar Variable on the left side of the reduction operation
5587/// (references element of array in original variable).
5588/// \param RHSVar Variable on the right side of the reduction operation
5589/// (references element of array in original variable).
5590/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5591/// RHSVar.
5592static void EmitOMPAggregateReduction(
5593 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5594 const VarDecl *RHSVar,
5595 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5596 const Expr *, const Expr *)> &RedOpGen,
5597 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5598 const Expr *UpExpr = nullptr) {
5599 // Perform element-by-element initialization.
5600 QualType ElementTy;
5601 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5602 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5603
5604 // Drill down to the base element type on both arrays.
5605 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5606 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5607
5608 llvm::Value *RHSBegin = RHSAddr.getPointer();
5609 llvm::Value *LHSBegin = LHSAddr.getPointer();
5610 // Cast from pointer to array type to pointer to single element.
5611 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5612 // The basic structure here is a while-do loop.
5613 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5614 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5615 llvm::Value *IsEmpty =
5616 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5617 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5618
5619 // Enter the loop body, making that address the current address.
5620 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5621 CGF.EmitBlock(BodyBB);
5622
5623 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5624
5625 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5626 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5627 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5628 Address RHSElementCurrent =
5629 Address(RHSElementPHI,
5630 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5631
5632 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5633 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5634 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5635 Address LHSElementCurrent =
5636 Address(LHSElementPHI,
5637 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5638
5639 // Emit copy.
5640 CodeGenFunction::OMPPrivateScope Scope(CGF);
5641 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5642 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5643 Scope.Privatize();
5644 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5645 Scope.ForceCleanup();
5646
5647 // Shift the address forward by one element.
5648 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5649 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5650 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5651 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5652 // Check whether we've reached the end.
5653 llvm::Value *Done =
5654 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5655 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5656 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5657 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5658
5659 // Done.
5660 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5661}
5662
5663/// Emit reduction combiner. If the combiner is a simple expression emit it as
5664/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5665/// UDR combiner function.
5666static void emitReductionCombiner(CodeGenFunction &CGF,
5667 const Expr *ReductionOp) {
5668 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5669 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5670 if (const auto *DRE =
5671 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5672 if (const auto *DRD =
5673 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5674 std::pair<llvm::Function *, llvm::Function *> Reduction =
5675 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5676 RValue Func = RValue::get(Reduction.first);
5677 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5678 CGF.EmitIgnoredExpr(ReductionOp);
5679 return;
5680 }
5681 CGF.EmitIgnoredExpr(ReductionOp);
5682}
5683
5684llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5685 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5686 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5687 ArrayRef<const Expr *> ReductionOps) {
5688 ASTContext &C = CGM.getContext();
5689
5690 // void reduction_func(void *LHSArg, void *RHSArg);
5691 FunctionArgList Args;
5692 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5693 ImplicitParamDecl::Other);
5694 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5695 ImplicitParamDecl::Other);
5696 Args.push_back(&LHSArg);
5697 Args.push_back(&RHSArg);
5698 const auto &CGFI =
5699 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5700 std::string Name = getName({"omp", "reduction", "reduction_func"});
5701 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5702 llvm::GlobalValue::InternalLinkage, Name,
5703 &CGM.getModule());
5704 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5705 Fn->setDoesNotRecurse();
5706 CodeGenFunction CGF(CGM);
5707 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5708
5709 // Dst = (void*[n])(LHSArg);
5710 // Src = (void*[n])(RHSArg);
5711 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5712 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5713 ArgsType), CGF.getPointerAlign());
5714 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5715 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5716 ArgsType), CGF.getPointerAlign());
5717
5718 // ...
5719 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5720 // ...
5721 CodeGenFunction::OMPPrivateScope Scope(CGF);
5722 auto IPriv = Privates.begin();
5723 unsigned Idx = 0;
5724 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5725 const auto *RHSVar =
5726 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5727 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5728 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5729 });
5730 const auto *LHSVar =
5731 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5732 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5733 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5734 });
5735 QualType PrivTy = (*IPriv)->getType();
5736 if (PrivTy->isVariablyModifiedType()) {
5737 // Get array size and emit VLA type.
5738 ++Idx;
5739 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5740 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5741 const VariableArrayType *VLA =
5742