Bug Summary

File:tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 2883, column 20
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn373517/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn373517/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn373517=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-02-234743-9763-1 -x c++ /build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGOpenMPRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/CodeGen/ConstantInitBuilder.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/StmtOpenMP.h"
21#include "clang/Basic/BitmaskEnum.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30
31using namespace clang;
32using namespace CodeGen;
33
34namespace {
35/// Base class for handling code generation inside OpenMP regions.
36class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
37public:
38 /// Kinds of OpenMP regions used in codegen.
39 enum CGOpenMPRegionKind {
40 /// Region with outlined function for standalone 'parallel'
41 /// directive.
42 ParallelOutlinedRegion,
43 /// Region with outlined function for standalone 'task' directive.
44 TaskOutlinedRegion,
45 /// Region for constructs that do not require function outlining,
46 /// like 'for', 'sections', 'atomic' etc. directives.
47 InlinedRegion,
48 /// Region with outlined function for standalone 'target' directive.
49 TargetRegion,
50 };
51
52 CGOpenMPRegionInfo(const CapturedStmt &CS,
53 const CGOpenMPRegionKind RegionKind,
54 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
55 bool HasCancel)
56 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58
59 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
60 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61 bool HasCancel)
62 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
64
65 /// Get a variable or parameter for storing global thread id
66 /// inside OpenMP construct.
67 virtual const VarDecl *getThreadIDVariable() const = 0;
68
69 /// Emit the captured statement body.
70 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
71
72 /// Get an LValue for the current ThreadID variable.
73 /// \return LValue for thread id variable. This LValue always has type int32*.
74 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75
76 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
77
78 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
79
80 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
81
82 bool hasCancel() const { return HasCancel; }
83
84 static bool classof(const CGCapturedStmtInfo *Info) {
85 return Info->getKind() == CR_OpenMP;
86 }
87
88 ~CGOpenMPRegionInfo() override = default;
89
90protected:
91 CGOpenMPRegionKind RegionKind;
92 RegionCodeGenTy CodeGen;
93 OpenMPDirectiveKind Kind;
94 bool HasCancel;
95};
96
97/// API for captured statement code generation in OpenMP constructs.
98class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
99public:
100 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
101 const RegionCodeGenTy &CodeGen,
102 OpenMPDirectiveKind Kind, bool HasCancel,
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
105 HasCancel),
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 107, __PRETTY_FUNCTION__))
;
108 }
109
110 /// Get a variable or parameter for storing global thread id
111 /// inside OpenMP construct.
112 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
113
114 /// Get the name of the capture helper.
115 StringRef getHelperName() const override { return HelperName; }
116
117 static bool classof(const CGCapturedStmtInfo *Info) {
118 return CGOpenMPRegionInfo::classof(Info) &&
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
121 }
122
123private:
124 /// A variable or parameter storing global thread id for OpenMP
125 /// constructs.
126 const VarDecl *ThreadIDVar;
127 StringRef HelperName;
128};
129
130/// API for captured statement code generation in OpenMP constructs.
131class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
132public:
133 class UntiedTaskActionTy final : public PrePostActionTy {
134 bool Untied;
135 const VarDecl *PartIDVar;
136 const RegionCodeGenTy UntiedCodeGen;
137 llvm::SwitchInst *UntiedSwitch = nullptr;
138
139 public:
140 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
141 const RegionCodeGenTy &UntiedCodeGen)
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
143 void Enter(CodeGenFunction &CGF) override {
144 if (Untied) {
145 // Emit task switching point.
146 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
147 CGF.GetAddrOfLocalVar(PartIDVar),
148 PartIDVar->getType()->castAs<PointerType>());
149 llvm::Value *Res =
150 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
151 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
152 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153 CGF.EmitBlock(DoneBB);
154 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157 CGF.Builder.GetInsertBlock());
158 emitUntiedSwitch(CGF);
159 }
160 }
161 void emitUntiedSwitch(CodeGenFunction &CGF) const {
162 if (Untied) {
163 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
164 CGF.GetAddrOfLocalVar(PartIDVar),
165 PartIDVar->getType()->castAs<PointerType>());
166 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167 PartIdLVal);
168 UntiedCodeGen(CGF);
169 CodeGenFunction::JumpDest CurPoint =
170 CGF.getJumpDestInCurrentScope(".untied.next.");
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174 CGF.Builder.GetInsertBlock());
175 CGF.EmitBranchThroughCleanup(CurPoint);
176 CGF.EmitBlock(CurPoint.getBlock());
177 }
178 }
179 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180 };
181 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
182 const VarDecl *ThreadIDVar,
183 const RegionCodeGenTy &CodeGen,
184 OpenMPDirectiveKind Kind, bool HasCancel,
185 const UntiedTaskActionTy &Action)
186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187 ThreadIDVar(ThreadIDVar), Action(Action) {
188 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 188, __PRETTY_FUNCTION__))
;
189 }
190
191 /// Get a variable or parameter for storing global thread id
192 /// inside OpenMP construct.
193 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
194
195 /// Get an LValue for the current ThreadID variable.
196 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197
198 /// Get the name of the capture helper.
199 StringRef getHelperName() const override { return ".omp_outlined."; }
200
201 void emitUntiedSwitch(CodeGenFunction &CGF) override {
202 Action.emitUntiedSwitch(CGF);
203 }
204
205 static bool classof(const CGCapturedStmtInfo *Info) {
206 return CGOpenMPRegionInfo::classof(Info) &&
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208 TaskOutlinedRegion;
209 }
210
211private:
212 /// A variable or parameter storing global thread id for OpenMP
213 /// constructs.
214 const VarDecl *ThreadIDVar;
215 /// Action for emitting code for untied tasks.
216 const UntiedTaskActionTy &Action;
217};
218
219/// API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
223 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
224 const RegionCodeGenTy &CodeGen,
225 OpenMPDirectiveKind Kind, bool HasCancel)
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OldCSI(OldCSI),
228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229
230 // Retrieve the value of the context parameter.
231 llvm::Value *getContextValue() const override {
232 if (OuterRegionInfo)
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 234)
;
235 }
236
237 void setContextValue(llvm::Value *V) override {
238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
240 return;
241 }
242 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 242)
;
243 }
244
245 /// Lookup the captured field decl for a variable.
246 const FieldDecl *lookup(const VarDecl *VD) const override {
247 if (OuterRegionInfo)
248 return OuterRegionInfo->lookup(VD);
249 // If there is no outer outlined region,no need to lookup in a list of
250 // captured variables, we can use the original one.
251 return nullptr;
252 }
253
254 FieldDecl *getThisFieldDecl() const override {
255 if (OuterRegionInfo)
256 return OuterRegionInfo->getThisFieldDecl();
257 return nullptr;
258 }
259
260 /// Get a variable or parameter for storing global thread id
261 /// inside OpenMP construct.
262 const VarDecl *getThreadIDVariable() const override {
263 if (OuterRegionInfo)
264 return OuterRegionInfo->getThreadIDVariable();
265 return nullptr;
266 }
267
268 /// Get an LValue for the current ThreadID variable.
269 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270 if (OuterRegionInfo)
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 272)
;
273 }
274
275 /// Get the name of the capture helper.
276 StringRef getHelperName() const override {
277 if (auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
279 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 279)
;
280 }
281
282 void emitUntiedSwitch(CodeGenFunction &CGF) override {
283 if (OuterRegionInfo)
284 OuterRegionInfo->emitUntiedSwitch(CGF);
285 }
286
287 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
289 static bool classof(const CGCapturedStmtInfo *Info) {
290 return CGOpenMPRegionInfo::classof(Info) &&
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292 }
293
294 ~CGOpenMPInlinedRegionInfo() override = default;
295
296private:
297 /// CodeGen info about outer OpenMP region.
298 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299 CGOpenMPRegionInfo *OuterRegionInfo;
300};
301
302/// API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
308public:
309 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
310 const RegionCodeGenTy &CodeGen, StringRef HelperName)
311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
312 /*HasCancel=*/false),
313 HelperName(HelperName) {}
314
315 /// This is unused for target regions because each starts executing
316 /// with a single thread.
317 const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319 /// Get the name of the capture helper.
320 StringRef getHelperName() const override { return HelperName; }
321
322 static bool classof(const CGCapturedStmtInfo *Info) {
323 return CGOpenMPRegionInfo::classof(Info) &&
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325 }
326
327private:
328 StringRef HelperName;
329};
330
331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
332 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 332)
;
333}
334/// API for generation of expressions captured in a innermost OpenMP
335/// region.
336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
337public:
338 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340 OMPD_unknown,
341 /*HasCancel=*/false),
342 PrivScope(CGF) {
343 // Make sure the globals captured in the provided statement are local by
344 // using the privatization logic. We assume the same variable is not
345 // captured more than once.
346 for (const auto &C : CS.captures()) {
347 if (!C.capturesVariable() && !C.capturesVariableByCopy())
348 continue;
349
350 const VarDecl *VD = C.getCapturedVar();
351 if (VD->isLocalVarDeclOrParm())
352 continue;
353
354 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
355 /*RefersToEnclosingVariableOrCapture=*/false,
356 VD->getType().getNonReferenceType(), VK_LValue,
357 C.getLocation());
358 PrivScope.addPrivate(
359 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
360 }
361 (void)PrivScope.Privatize();
362 }
363
364 /// Lookup the captured field decl for a variable.
365 const FieldDecl *lookup(const VarDecl *VD) const override {
366 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
367 return FD;
368 return nullptr;
369 }
370
371 /// Emit the captured statement body.
372 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
373 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 373)
;
374 }
375
376 /// Get a variable or parameter for storing global thread id
377 /// inside OpenMP construct.
378 const VarDecl *getThreadIDVariable() const override {
379 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 379)
;
380 }
381
382 /// Get the name of the capture helper.
383 StringRef getHelperName() const override {
384 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 384)
;
385 }
386
387 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
388
389private:
390 /// Private scope to capture global variables.
391 CodeGenFunction::OMPPrivateScope PrivScope;
392};
393
394/// RAII for emitting code of OpenMP constructs.
395class InlinedOpenMPRegionRAII {
396 CodeGenFunction &CGF;
397 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398 FieldDecl *LambdaThisCaptureField = nullptr;
399 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
400
401public:
402 /// Constructs region for combined constructs.
403 /// \param CodeGen Code generation sequence for combined directives. Includes
404 /// a list of functions used for code generation of implicitly inlined
405 /// regions.
406 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
407 OpenMPDirectiveKind Kind, bool HasCancel)
408 : CGF(CGF) {
409 // Start emission for the construct.
410 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414 CGF.LambdaThisCaptureField = nullptr;
415 BlockInfo = CGF.BlockInfo;
416 CGF.BlockInfo = nullptr;
417 }
418
419 ~InlinedOpenMPRegionRAII() {
420 // Restore original CapturedStmtInfo only if we're done with code emission.
421 auto *OldCSI =
422 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
423 delete CGF.CapturedStmtInfo;
424 CGF.CapturedStmtInfo = OldCSI;
425 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
426 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
427 CGF.BlockInfo = BlockInfo;
428 }
429};
430
431/// Values for bit flags used in the ident_t to describe the fields.
432/// All enumeric elements are named and described in accordance with the code
433/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
434enum OpenMPLocationFlags : unsigned {
435 /// Use trampoline for internal microtask.
436 OMP_IDENT_IMD = 0x01,
437 /// Use c-style ident structure.
438 OMP_IDENT_KMPC = 0x02,
439 /// Atomic reduction option for kmpc_reduce.
440 OMP_ATOMIC_REDUCE = 0x10,
441 /// Explicit 'barrier' directive.
442 OMP_IDENT_BARRIER_EXPL = 0x20,
443 /// Implicit barrier in code.
444 OMP_IDENT_BARRIER_IMPL = 0x40,
445 /// Implicit barrier in 'for' directive.
446 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
447 /// Implicit barrier in 'sections' directive.
448 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
449 /// Implicit barrier in 'single' directive.
450 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
451 /// Call of __kmp_for_static_init for static loop.
452 OMP_IDENT_WORK_LOOP = 0x200,
453 /// Call of __kmp_for_static_init for sections.
454 OMP_IDENT_WORK_SECTIONS = 0x400,
455 /// Call of __kmp_for_static_init for distribute.
456 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
458};
459
460namespace {
461LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
462/// Values for bit flags for marking which requires clauses have been used.
463enum OpenMPOffloadingRequiresDirFlags : int64_t {
464 /// flag undefined.
465 OMP_REQ_UNDEFINED = 0x000,
466 /// no requires clause present.
467 OMP_REQ_NONE = 0x001,
468 /// reverse_offload clause.
469 OMP_REQ_REVERSE_OFFLOAD = 0x002,
470 /// unified_address clause.
471 OMP_REQ_UNIFIED_ADDRESS = 0x004,
472 /// unified_shared_memory clause.
473 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
474 /// dynamic_allocators clause.
475 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
476 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
477};
478
479enum OpenMPOffloadingReservedDeviceIDs {
480 /// Device ID if the device was not defined, runtime should get it
481 /// from environment variables in the spec.
482 OMP_DEVICEID_UNDEF = -1,
483};
484} // anonymous namespace
485
486/// Describes ident structure that describes a source location.
487/// All descriptions are taken from
488/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
489/// Original structure:
490/// typedef struct ident {
491/// kmp_int32 reserved_1; /**< might be used in Fortran;
492/// see above */
493/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
494/// KMP_IDENT_KMPC identifies this union
495/// member */
496/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
497/// see above */
498///#if USE_ITT_BUILD
499/// /* but currently used for storing
500/// region-specific ITT */
501/// /* contextual information. */
502///#endif /* USE_ITT_BUILD */
503/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
504/// C++ */
505/// char const *psource; /**< String describing the source location.
506/// The string is composed of semi-colon separated
507// fields which describe the source file,
508/// the function and a pair of line numbers that
509/// delimit the construct.
510/// */
511/// } ident_t;
512enum IdentFieldIndex {
513 /// might be used in Fortran
514 IdentField_Reserved_1,
515 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
516 IdentField_Flags,
517 /// Not really used in Fortran any more
518 IdentField_Reserved_2,
519 /// Source[4] in Fortran, do not use for C++
520 IdentField_Reserved_3,
521 /// String describing the source location. The string is composed of
522 /// semi-colon separated fields which describe the source file, the function
523 /// and a pair of line numbers that delimit the construct.
524 IdentField_PSource
525};
526
527/// Schedule types for 'omp for' loops (these enumerators are taken from
528/// the enum sched_type in kmp.h).
529enum OpenMPSchedType {
530 /// Lower bound for default (unordered) versions.
531 OMP_sch_lower = 32,
532 OMP_sch_static_chunked = 33,
533 OMP_sch_static = 34,
534 OMP_sch_dynamic_chunked = 35,
535 OMP_sch_guided_chunked = 36,
536 OMP_sch_runtime = 37,
537 OMP_sch_auto = 38,
538 /// static with chunk adjustment (e.g., simd)
539 OMP_sch_static_balanced_chunked = 45,
540 /// Lower bound for 'ordered' versions.
541 OMP_ord_lower = 64,
542 OMP_ord_static_chunked = 65,
543 OMP_ord_static = 66,
544 OMP_ord_dynamic_chunked = 67,
545 OMP_ord_guided_chunked = 68,
546 OMP_ord_runtime = 69,
547 OMP_ord_auto = 70,
548 OMP_sch_default = OMP_sch_static,
549 /// dist_schedule types
550 OMP_dist_sch_static_chunked = 91,
551 OMP_dist_sch_static = 92,
552 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
553 /// Set if the monotonic schedule modifier was present.
554 OMP_sch_modifier_monotonic = (1 << 29),
555 /// Set if the nonmonotonic schedule modifier was present.
556 OMP_sch_modifier_nonmonotonic = (1 << 30),
557};
558
559enum OpenMPRTLFunction {
560 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
561 /// kmpc_micro microtask, ...);
562 OMPRTL__kmpc_fork_call,
563 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
564 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
565 OMPRTL__kmpc_threadprivate_cached,
566 /// Call to void __kmpc_threadprivate_register( ident_t *,
567 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
568 OMPRTL__kmpc_threadprivate_register,
569 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
570 OMPRTL__kmpc_global_thread_num,
571 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
572 // kmp_critical_name *crit);
573 OMPRTL__kmpc_critical,
574 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
575 // global_tid, kmp_critical_name *crit, uintptr_t hint);
576 OMPRTL__kmpc_critical_with_hint,
577 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
578 // kmp_critical_name *crit);
579 OMPRTL__kmpc_end_critical,
580 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
581 // global_tid);
582 OMPRTL__kmpc_cancel_barrier,
583 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
584 OMPRTL__kmpc_barrier,
585 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
586 OMPRTL__kmpc_for_static_fini,
587 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
588 // global_tid);
589 OMPRTL__kmpc_serialized_parallel,
590 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
591 // global_tid);
592 OMPRTL__kmpc_end_serialized_parallel,
593 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
594 // kmp_int32 num_threads);
595 OMPRTL__kmpc_push_num_threads,
596 // Call to void __kmpc_flush(ident_t *loc);
597 OMPRTL__kmpc_flush,
598 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
599 OMPRTL__kmpc_master,
600 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
601 OMPRTL__kmpc_end_master,
602 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
603 // int end_part);
604 OMPRTL__kmpc_omp_taskyield,
605 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
606 OMPRTL__kmpc_single,
607 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
608 OMPRTL__kmpc_end_single,
609 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
610 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
611 // kmp_routine_entry_t *task_entry);
612 OMPRTL__kmpc_omp_task_alloc,
613 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
614 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
615 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
616 // kmp_int64 device_id);
617 OMPRTL__kmpc_omp_target_task_alloc,
618 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
619 // new_task);
620 OMPRTL__kmpc_omp_task,
621 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
622 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
623 // kmp_int32 didit);
624 OMPRTL__kmpc_copyprivate,
625 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
626 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
627 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
628 OMPRTL__kmpc_reduce,
629 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
630 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
631 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
632 // *lck);
633 OMPRTL__kmpc_reduce_nowait,
634 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
635 // kmp_critical_name *lck);
636 OMPRTL__kmpc_end_reduce,
637 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
638 // kmp_critical_name *lck);
639 OMPRTL__kmpc_end_reduce_nowait,
640 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
641 // kmp_task_t * new_task);
642 OMPRTL__kmpc_omp_task_begin_if0,
643 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
644 // kmp_task_t * new_task);
645 OMPRTL__kmpc_omp_task_complete_if0,
646 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
647 OMPRTL__kmpc_ordered,
648 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
649 OMPRTL__kmpc_end_ordered,
650 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
651 // global_tid);
652 OMPRTL__kmpc_omp_taskwait,
653 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
654 OMPRTL__kmpc_taskgroup,
655 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
656 OMPRTL__kmpc_end_taskgroup,
657 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
658 // int proc_bind);
659 OMPRTL__kmpc_push_proc_bind,
660 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
661 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
662 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
663 OMPRTL__kmpc_omp_task_with_deps,
664 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
665 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
666 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
667 OMPRTL__kmpc_omp_wait_deps,
668 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
669 // global_tid, kmp_int32 cncl_kind);
670 OMPRTL__kmpc_cancellationpoint,
671 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
672 // kmp_int32 cncl_kind);
673 OMPRTL__kmpc_cancel,
674 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
675 // kmp_int32 num_teams, kmp_int32 thread_limit);
676 OMPRTL__kmpc_push_num_teams,
677 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
678 // microtask, ...);
679 OMPRTL__kmpc_fork_teams,
680 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
681 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
682 // sched, kmp_uint64 grainsize, void *task_dup);
683 OMPRTL__kmpc_taskloop,
684 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
685 // num_dims, struct kmp_dim *dims);
686 OMPRTL__kmpc_doacross_init,
687 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
688 OMPRTL__kmpc_doacross_fini,
689 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
690 // *vec);
691 OMPRTL__kmpc_doacross_post,
692 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
693 // *vec);
694 OMPRTL__kmpc_doacross_wait,
695 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
696 // *data);
697 OMPRTL__kmpc_task_reduction_init,
698 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
699 // *d);
700 OMPRTL__kmpc_task_reduction_get_th_data,
701 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
702 OMPRTL__kmpc_alloc,
703 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
704 OMPRTL__kmpc_free,
705
706 //
707 // Offloading related calls
708 //
709 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
710 // size);
711 OMPRTL__kmpc_push_target_tripcount,
712 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
713 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
714 // *arg_types);
715 OMPRTL__tgt_target,
716 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
717 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
718 // *arg_types);
719 OMPRTL__tgt_target_nowait,
720 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
721 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
722 // *arg_types, int32_t num_teams, int32_t thread_limit);
723 OMPRTL__tgt_target_teams,
724 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
725 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
726 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
727 OMPRTL__tgt_target_teams_nowait,
728 // Call to void __tgt_register_requires(int64_t flags);
729 OMPRTL__tgt_register_requires,
730 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
731 OMPRTL__tgt_register_lib,
732 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
733 OMPRTL__tgt_unregister_lib,
734 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
735 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
736 OMPRTL__tgt_target_data_begin,
737 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
738 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
739 // *arg_types);
740 OMPRTL__tgt_target_data_begin_nowait,
741 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
742 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
743 OMPRTL__tgt_target_data_end,
744 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
745 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
746 // *arg_types);
747 OMPRTL__tgt_target_data_end_nowait,
748 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
749 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
750 OMPRTL__tgt_target_data_update,
751 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
752 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
753 // *arg_types);
754 OMPRTL__tgt_target_data_update_nowait,
755 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
756 OMPRTL__tgt_mapper_num_components,
757 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
758 // *base, void *begin, int64_t size, int64_t type);
759 OMPRTL__tgt_push_mapper_component,
760};
761
762/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
763/// region.
764class CleanupTy final : public EHScopeStack::Cleanup {
765 PrePostActionTy *Action;
766
767public:
768 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
769 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
770 if (!CGF.HaveInsertPoint())
771 return;
772 Action->Exit(CGF);
773 }
774};
775
776} // anonymous namespace
777
778void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
779 CodeGenFunction::RunCleanupsScope Scope(CGF);
780 if (PrePostAction) {
781 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
782 Callback(CodeGen, CGF, *PrePostAction);
783 } else {
784 PrePostActionTy Action;
785 Callback(CodeGen, CGF, Action);
786 }
787}
788
789/// Check if the combiner is a call to UDR combiner and if it is so return the
790/// UDR decl used for reduction.
791static const OMPDeclareReductionDecl *
792getReductionInit(const Expr *ReductionOp) {
793 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
794 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
795 if (const auto *DRE =
796 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
797 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
798 return DRD;
799 return nullptr;
800}
801
802static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
803 const OMPDeclareReductionDecl *DRD,
804 const Expr *InitOp,
805 Address Private, Address Original,
806 QualType Ty) {
807 if (DRD->getInitializer()) {
808 std::pair<llvm::Function *, llvm::Function *> Reduction =
809 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
810 const auto *CE = cast<CallExpr>(InitOp);
811 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
812 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
813 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
814 const auto *LHSDRE =
815 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
816 const auto *RHSDRE =
817 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
818 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
819 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
820 [=]() { return Private; });
821 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
822 [=]() { return Original; });
823 (void)PrivateScope.Privatize();
824 RValue Func = RValue::get(Reduction.second);
825 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
826 CGF.EmitIgnoredExpr(InitOp);
827 } else {
828 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
829 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
830 auto *GV = new llvm::GlobalVariable(
831 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
832 llvm::GlobalValue::PrivateLinkage, Init, Name);
833 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
834 RValue InitRVal;
835 switch (CGF.getEvaluationKind(Ty)) {
836 case TEK_Scalar:
837 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
838 break;
839 case TEK_Complex:
840 InitRVal =
841 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
842 break;
843 case TEK_Aggregate:
844 InitRVal = RValue::getAggregate(LV.getAddress());
845 break;
846 }
847 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
848 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
849 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
850 /*IsInitializer=*/false);
851 }
852}
853
854/// Emit initialization of arrays of complex types.
855/// \param DestAddr Address of the array.
856/// \param Type Type of array.
857/// \param Init Initial expression of array.
858/// \param SrcAddr Address of the original array.
859static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
860 QualType Type, bool EmitDeclareReductionInit,
861 const Expr *Init,
862 const OMPDeclareReductionDecl *DRD,
863 Address SrcAddr = Address::invalid()) {
864 // Perform element-by-element initialization.
865 QualType ElementTy;
866
867 // Drill down to the base element type on both arrays.
868 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
869 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
870 DestAddr =
871 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
872 if (DRD)
873 SrcAddr =
874 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
875
876 llvm::Value *SrcBegin = nullptr;
877 if (DRD)
878 SrcBegin = SrcAddr.getPointer();
879 llvm::Value *DestBegin = DestAddr.getPointer();
880 // Cast from pointer to array type to pointer to single element.
881 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
882 // The basic structure here is a while-do loop.
883 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
884 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
885 llvm::Value *IsEmpty =
886 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
887 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
888
889 // Enter the loop body, making that address the current address.
890 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
891 CGF.EmitBlock(BodyBB);
892
893 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
894
895 llvm::PHINode *SrcElementPHI = nullptr;
896 Address SrcElementCurrent = Address::invalid();
897 if (DRD) {
898 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
899 "omp.arraycpy.srcElementPast");
900 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
901 SrcElementCurrent =
902 Address(SrcElementPHI,
903 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
904 }
905 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
906 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
907 DestElementPHI->addIncoming(DestBegin, EntryBB);
908 Address DestElementCurrent =
909 Address(DestElementPHI,
910 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
911
912 // Emit copy.
913 {
914 CodeGenFunction::RunCleanupsScope InitScope(CGF);
915 if (EmitDeclareReductionInit) {
916 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
917 SrcElementCurrent, ElementTy);
918 } else
919 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
920 /*IsInitializer=*/false);
921 }
922
923 if (DRD) {
924 // Shift the address forward by one element.
925 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
926 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
927 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
928 }
929
930 // Shift the address forward by one element.
931 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
932 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
933 // Check whether we've reached the end.
934 llvm::Value *Done =
935 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
936 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
937 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
938
939 // Done.
940 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
941}
942
943LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
944 return CGF.EmitOMPSharedLValue(E);
945}
946
947LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
948 const Expr *E) {
949 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
950 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
951 return LValue();
952}
953
954void ReductionCodeGen::emitAggregateInitialization(
955 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
956 const OMPDeclareReductionDecl *DRD) {
957 // Emit VarDecl with copy init for arrays.
958 // Get the address of the original variable captured in current
959 // captured region.
960 const auto *PrivateVD =
961 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
962 bool EmitDeclareReductionInit =
963 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
964 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
965 EmitDeclareReductionInit,
966 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
967 : PrivateVD->getInit(),
968 DRD, SharedLVal.getAddress());
969}
970
971ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
972 ArrayRef<const Expr *> Privates,
973 ArrayRef<const Expr *> ReductionOps) {
974 ClausesData.reserve(Shareds.size());
975 SharedAddresses.reserve(Shareds.size());
976 Sizes.reserve(Shareds.size());
977 BaseDecls.reserve(Shareds.size());
978 auto IPriv = Privates.begin();
979 auto IRed = ReductionOps.begin();
980 for (const Expr *Ref : Shareds) {
981 ClausesData.emplace_back(Ref, *IPriv, *IRed);
982 std::advance(IPriv, 1);
983 std::advance(IRed, 1);
984 }
985}
986
987void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
988 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
989 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 989, __PRETTY_FUNCTION__))
;
990 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
991 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
992 SharedAddresses.emplace_back(First, Second);
993}
994
995void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
996 const auto *PrivateVD =
997 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
998 QualType PrivateType = PrivateVD->getType();
999 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1000 if (!PrivateType->isVariablyModifiedType()) {
1001 Sizes.emplace_back(
1002 CGF.getTypeSize(
1003 SharedAddresses[N].first.getType().getNonReferenceType()),
1004 nullptr);
1005 return;
1006 }
1007 llvm::Value *Size;
1008 llvm::Value *SizeInChars;
1009 auto *ElemType =
1010 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
1011 ->getElementType();
1012 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1013 if (AsArraySection) {
1014 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
1015 SharedAddresses[N].first.getPointer());
1016 Size = CGF.Builder.CreateNUWAdd(
1017 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1018 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1019 } else {
1020 SizeInChars = CGF.getTypeSize(
1021 SharedAddresses[N].first.getType().getNonReferenceType());
1022 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1023 }
1024 Sizes.emplace_back(SizeInChars, Size);
1025 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1026 CGF,
1027 cast<OpaqueValueExpr>(
1028 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1029 RValue::get(Size));
1030 CGF.EmitVariablyModifiedType(PrivateType);
1031}
1032
1033void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1034 llvm::Value *Size) {
1035 const auto *PrivateVD =
1036 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1037 QualType PrivateType = PrivateVD->getType();
1038 if (!PrivateType->isVariablyModifiedType()) {
1039 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1040 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
1041 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1041, __PRETTY_FUNCTION__))
;
1042 return;
1043 }
1044 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1045 CGF,
1046 cast<OpaqueValueExpr>(
1047 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1048 RValue::get(Size));
1049 CGF.EmitVariablyModifiedType(PrivateType);
1050}
1051
1052void ReductionCodeGen::emitInitialization(
1053 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1054 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1055 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1055, __PRETTY_FUNCTION__))
;
1056 const auto *PrivateVD =
1057 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1058 const OMPDeclareReductionDecl *DRD =
1059 getReductionInit(ClausesData[N].ReductionOp);
1060 QualType PrivateType = PrivateVD->getType();
1061 PrivateAddr = CGF.Builder.CreateElementBitCast(
1062 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1063 QualType SharedType = SharedAddresses[N].first.getType();
1064 SharedLVal = CGF.MakeAddrLValue(
1065 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1066 CGF.ConvertTypeForMem(SharedType)),
1067 SharedType, SharedAddresses[N].first.getBaseInfo(),
1068 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1069 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1070 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1071 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1072 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1073 PrivateAddr, SharedLVal.getAddress(),
1074 SharedLVal.getType());
1075 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1076 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1077 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1078 PrivateVD->getType().getQualifiers(),
1079 /*IsInitializer=*/false);
1080 }
1081}
1082
1083bool ReductionCodeGen::needCleanups(unsigned N) {
1084 const auto *PrivateVD =
1085 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1086 QualType PrivateType = PrivateVD->getType();
1087 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1088 return DTorKind != QualType::DK_none;
1089}
1090
1091void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1092 Address PrivateAddr) {
1093 const auto *PrivateVD =
1094 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1095 QualType PrivateType = PrivateVD->getType();
1096 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1097 if (needCleanups(N)) {
1098 PrivateAddr = CGF.Builder.CreateElementBitCast(
1099 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1100 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1101 }
1102}
1103
1104static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1105 LValue BaseLV) {
1106 BaseTy = BaseTy.getNonReferenceType();
1107 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1108 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1109 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1110 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1111 } else {
1112 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1113 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1114 }
1115 BaseTy = BaseTy->getPointeeType();
1116 }
1117 return CGF.MakeAddrLValue(
1118 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1119 CGF.ConvertTypeForMem(ElTy)),
1120 BaseLV.getType(), BaseLV.getBaseInfo(),
1121 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1122}
1123
1124static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1125 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1126 llvm::Value *Addr) {
1127 Address Tmp = Address::invalid();
1128 Address TopTmp = Address::invalid();
1129 Address MostTopTmp = Address::invalid();
1130 BaseTy = BaseTy.getNonReferenceType();
1131 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1132 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1133 Tmp = CGF.CreateMemTemp(BaseTy);
1134 if (TopTmp.isValid())
1135 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1136 else
1137 MostTopTmp = Tmp;
1138 TopTmp = Tmp;
1139 BaseTy = BaseTy->getPointeeType();
1140 }
1141 llvm::Type *Ty = BaseLVType;
1142 if (Tmp.isValid())
1143 Ty = Tmp.getElementType();
1144 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1145 if (Tmp.isValid()) {
1146 CGF.Builder.CreateStore(Addr, Tmp);
1147 return MostTopTmp;
1148 }
1149 return Address(Addr, BaseLVAlignment);
1150}
1151
1152static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1153 const VarDecl *OrigVD = nullptr;
1154 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1155 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1156 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1157 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1158 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1159 Base = TempASE->getBase()->IgnoreParenImpCasts();
1160 DE = cast<DeclRefExpr>(Base);
1161 OrigVD = cast<VarDecl>(DE->getDecl());
1162 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1163 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1164 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1165 Base = TempASE->getBase()->IgnoreParenImpCasts();
1166 DE = cast<DeclRefExpr>(Base);
1167 OrigVD = cast<VarDecl>(DE->getDecl());
1168 }
1169 return OrigVD;
1170}
1171
1172Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1173 Address PrivateAddr) {
1174 const DeclRefExpr *DE;
1175 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1176 BaseDecls.emplace_back(OrigVD);
1177 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1178 LValue BaseLValue =
1179 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1180 OriginalBaseLValue);
1181 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1182 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1183 llvm::Value *PrivatePointer =
1184 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1185 PrivateAddr.getPointer(),
1186 SharedAddresses[N].first.getAddress().getType());
1187 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1188 return castToBase(CGF, OrigVD->getType(),
1189 SharedAddresses[N].first.getType(),
1190 OriginalBaseLValue.getAddress().getType(),
1191 OriginalBaseLValue.getAlignment(), Ptr);
1192 }
1193 BaseDecls.emplace_back(
1194 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1195 return PrivateAddr;
1196}
1197
1198bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1199 const OMPDeclareReductionDecl *DRD =
1200 getReductionInit(ClausesData[N].ReductionOp);
1201 return DRD && DRD->getInitializer();
1202}
1203
1204LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1205 return CGF.EmitLoadOfPointerLValue(
1206 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1207 getThreadIDVariable()->getType()->castAs<PointerType>());
1208}
1209
1210void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1211 if (!CGF.HaveInsertPoint())
1212 return;
1213 // 1.2.2 OpenMP Language Terminology
1214 // Structured block - An executable statement with a single entry at the
1215 // top and a single exit at the bottom.
1216 // The point of exit cannot be a branch out of the structured block.
1217 // longjmp() and throw() must not violate the entry/exit criteria.
1218 CGF.EHStack.pushTerminate();
1219 CodeGen(CGF);
1220 CGF.EHStack.popTerminate();
1221}
1222
1223LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1224 CodeGenFunction &CGF) {
1225 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1226 getThreadIDVariable()->getType(),
1227 AlignmentSource::Decl);
1228}
1229
1230static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1231 QualType FieldTy) {
1232 auto *Field = FieldDecl::Create(
1233 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1234 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1235 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1236 Field->setAccess(AS_public);
1237 DC->addDecl(Field);
1238 return Field;
1239}
1240
1241CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1242 StringRef Separator)
1243 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1244 OffloadEntriesInfoManager(CGM) {
1245 ASTContext &C = CGM.getContext();
1246 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1247 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1248 RD->startDefinition();
1249 // reserved_1
1250 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1251 // flags
1252 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1253 // reserved_2
1254 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1255 // reserved_3
1256 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1257 // psource
1258 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1259 RD->completeDefinition();
1260 IdentQTy = C.getRecordType(RD);
1261 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1262 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1263
1264 loadOffloadInfoMetadata();
1265}
1266
1267static bool tryEmitAlias(CodeGenModule &CGM, const GlobalDecl &NewGD,
1268 const GlobalDecl &OldGD, llvm::GlobalValue *OrigAddr,
1269 bool IsForDefinition) {
1270 // Emit at least a definition for the aliasee if the the address of the
1271 // original function is requested.
1272 if (IsForDefinition || OrigAddr)
1273 (void)CGM.GetAddrOfGlobal(NewGD);
1274 StringRef NewMangledName = CGM.getMangledName(NewGD);
1275 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1276 if (Addr && !Addr->isDeclaration()) {
1277 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1278 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD);
1279 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1280
1281 // Create a reference to the named value. This ensures that it is emitted
1282 // if a deferred decl.
1283 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1284
1285 // Create the new alias itself, but don't set a name yet.
1286 auto *GA =
1287 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1288
1289 if (OrigAddr) {
1290 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1290, __PRETTY_FUNCTION__))
;
1291
1292 GA->takeName(OrigAddr);
1293 OrigAddr->replaceAllUsesWith(
1294 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1295 OrigAddr->eraseFromParent();
1296 } else {
1297 GA->setName(CGM.getMangledName(OldGD));
1298 }
1299
1300 // Set attributes which are particular to an alias; this is a
1301 // specialization of the attributes which may be set on a global function.
1302 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1303 D->isWeakImported())
1304 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1305
1306 CGM.SetCommonAttributes(OldGD, GA);
1307 return true;
1308 }
1309 return false;
1310}
1311
1312void CGOpenMPRuntime::clear() {
1313 InternalVars.clear();
1314 // Clean non-target variable declarations possibly used only in debug info.
1315 for (const auto &Data : EmittedNonTargetVariables) {
1316 if (!Data.getValue().pointsToAliveValue())
1317 continue;
1318 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1319 if (!GV)
1320 continue;
1321 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1322 continue;
1323 GV->eraseFromParent();
1324 }
1325 // Emit aliases for the deferred aliasees.
1326 for (const auto &Pair : DeferredVariantFunction) {
1327 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1328 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1329 // If not able to emit alias, just emit original declaration.
1330 (void)tryEmitAlias(CGM, Pair.second.first, Pair.second.second, Addr,
1331 /*IsForDefinition=*/false);
1332 }
1333}
1334
1335std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1336 SmallString<128> Buffer;
1337 llvm::raw_svector_ostream OS(Buffer);
1338 StringRef Sep = FirstSeparator;
1339 for (StringRef Part : Parts) {
1340 OS << Sep << Part;
1341 Sep = Separator;
1342 }
1343 return OS.str();
1344}
1345
1346static llvm::Function *
1347emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1348 const Expr *CombinerInitializer, const VarDecl *In,
1349 const VarDecl *Out, bool IsCombiner) {
1350 // void .omp_combiner.(Ty *in, Ty *out);
1351 ASTContext &C = CGM.getContext();
1352 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1353 FunctionArgList Args;
1354 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1355 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1356 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1357 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1358 Args.push_back(&OmpOutParm);
1359 Args.push_back(&OmpInParm);
1360 const CGFunctionInfo &FnInfo =
1361 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1362 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1363 std::string Name = CGM.getOpenMPRuntime().getName(
1364 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1365 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1366 Name, &CGM.getModule());
1367 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1368 if (CGM.getLangOpts().Optimize) {
1369 Fn->removeFnAttr(llvm::Attribute::NoInline);
1370 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1371 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1372 }
1373 CodeGenFunction CGF(CGM);
1374 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1375 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1376 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1377 Out->getLocation());
1378 CodeGenFunction::OMPPrivateScope Scope(CGF);
1379 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1380 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1381 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1382 .getAddress();
1383 });
1384 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1385 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1386 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1387 .getAddress();
1388 });
1389 (void)Scope.Privatize();
1390 if (!IsCombiner && Out->hasInit() &&
1391 !CGF.isTrivialInitializer(Out->getInit())) {
1392 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1393 Out->getType().getQualifiers(),
1394 /*IsInitializer=*/true);
1395 }
1396 if (CombinerInitializer)
1397 CGF.EmitIgnoredExpr(CombinerInitializer);
1398 Scope.ForceCleanup();
1399 CGF.FinishFunction();
1400 return Fn;
1401}
1402
1403void CGOpenMPRuntime::emitUserDefinedReduction(
1404 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1405 if (UDRMap.count(D) > 0)
1406 return;
1407 llvm::Function *Combiner = emitCombinerOrInitializer(
1408 CGM, D->getType(), D->getCombiner(),
1409 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1410 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1411 /*IsCombiner=*/true);
1412 llvm::Function *Initializer = nullptr;
1413 if (const Expr *Init = D->getInitializer()) {
1414 Initializer = emitCombinerOrInitializer(
1415 CGM, D->getType(),
1416 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1417 : nullptr,
1418 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1419 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1420 /*IsCombiner=*/false);
1421 }
1422 UDRMap.try_emplace(D, Combiner, Initializer);
1423 if (CGF) {
1424 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1425 Decls.second.push_back(D);
1426 }
1427}
1428
1429std::pair<llvm::Function *, llvm::Function *>
1430CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1431 auto I = UDRMap.find(D);
1432 if (I != UDRMap.end())
1433 return I->second;
1434 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1435 return UDRMap.lookup(D);
1436}
1437
1438static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1439 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1440 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1441 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1442 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1443, __PRETTY_FUNCTION__))
1443 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1443, __PRETTY_FUNCTION__))
;
1444 CodeGenFunction CGF(CGM, true);
1445 bool HasCancel = false;
1446 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1447 HasCancel = OPD->hasCancel();
1448 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1449 HasCancel = OPSD->hasCancel();
1450 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1451 HasCancel = OPFD->hasCancel();
1452 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1453 HasCancel = OPFD->hasCancel();
1454 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1455 HasCancel = OPFD->hasCancel();
1456 else if (const auto *OPFD =
1457 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1458 HasCancel = OPFD->hasCancel();
1459 else if (const auto *OPFD =
1460 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1461 HasCancel = OPFD->hasCancel();
1462 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1463 HasCancel, OutlinedHelperName);
1464 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1465 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1466}
1467
1468llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1469 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1470 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1471 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1472 return emitParallelOrTeamsOutlinedFunction(
1473 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1474}
1475
1476llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1477 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1478 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1479 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1480 return emitParallelOrTeamsOutlinedFunction(
1481 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1482}
1483
1484llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1485 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1486 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1487 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1488 bool Tied, unsigned &NumberOfParts) {
1489 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1490 PrePostActionTy &) {
1491 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1492 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1493 llvm::Value *TaskArgs[] = {
1494 UpLoc, ThreadID,
1495 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1496 TaskTVar->getType()->castAs<PointerType>())
1497 .getPointer()};
1498 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1499 };
1500 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1501 UntiedCodeGen);
1502 CodeGen.setAction(Action);
1503 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1504, __PRETTY_FUNCTION__))
1504 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1504, __PRETTY_FUNCTION__))
;
1505 const OpenMPDirectiveKind Region =
1506 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1507 : OMPD_task;
1508 const CapturedStmt *CS = D.getCapturedStmt(Region);
1509 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1510 CodeGenFunction CGF(CGM, true);
1511 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1512 InnermostKind,
1513 TD ? TD->hasCancel() : false, Action);
1514 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1515 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1516 if (!Tied)
1517 NumberOfParts = Action.getNumberOfParts();
1518 return Res;
1519}
1520
1521static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1522 const RecordDecl *RD, const CGRecordLayout &RL,
1523 ArrayRef<llvm::Constant *> Data) {
1524 llvm::StructType *StructTy = RL.getLLVMType();
1525 unsigned PrevIdx = 0;
1526 ConstantInitBuilder CIBuilder(CGM);
1527 auto DI = Data.begin();
1528 for (const FieldDecl *FD : RD->fields()) {
1529 unsigned Idx = RL.getLLVMFieldNo(FD);
1530 // Fill the alignment.
1531 for (unsigned I = PrevIdx; I < Idx; ++I)
1532 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1533 PrevIdx = Idx + 1;
1534 Fields.add(*DI);
1535 ++DI;
1536 }
1537}
1538
1539template <class... As>
1540static llvm::GlobalVariable *
1541createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1542 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1543 As &&... Args) {
1544 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1545 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1546 ConstantInitBuilder CIBuilder(CGM);
1547 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1548 buildStructValue(Fields, CGM, RD, RL, Data);
1549 return Fields.finishAndCreateGlobal(
1550 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1551 std::forward<As>(Args)...);
1552}
1553
1554template <typename T>
1555static void
1556createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1557 ArrayRef<llvm::Constant *> Data,
1558 T &Parent) {
1559 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1560 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1561 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1562 buildStructValue(Fields, CGM, RD, RL, Data);
1563 Fields.finishAndAddTo(Parent);
1564}
1565
1566Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1567 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1568 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1569 FlagsTy FlagsKey(Flags, Reserved2Flags);
1570 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1571 if (!Entry) {
1572 if (!DefaultOpenMPPSource) {
1573 // Initialize default location for psource field of ident_t structure of
1574 // all ident_t objects. Format is ";file;function;line;column;;".
1575 // Taken from
1576 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1577 DefaultOpenMPPSource =
1578 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1579 DefaultOpenMPPSource =
1580 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1581 }
1582
1583 llvm::Constant *Data[] = {
1584 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1585 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1586 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1587 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1588 llvm::GlobalValue *DefaultOpenMPLocation =
1589 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1590 llvm::GlobalValue::PrivateLinkage);
1591 DefaultOpenMPLocation->setUnnamedAddr(
1592 llvm::GlobalValue::UnnamedAddr::Global);
1593
1594 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1595 }
1596 return Address(Entry, Align);
1597}
1598
1599void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1600 bool AtCurrentPoint) {
1601 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1602 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1602, __PRETTY_FUNCTION__))
;
1603
1604 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1605 if (AtCurrentPoint) {
1606 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1607 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1608 } else {
1609 Elem.second.ServiceInsertPt =
1610 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1611 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1612 }
1613}
1614
1615void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1616 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1617 if (Elem.second.ServiceInsertPt) {
1618 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1619 Elem.second.ServiceInsertPt = nullptr;
1620 Ptr->eraseFromParent();
1621 }
1622}
1623
1624llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1625 SourceLocation Loc,
1626 unsigned Flags) {
1627 Flags |= OMP_IDENT_KMPC;
1628 // If no debug info is generated - return global default location.
1629 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1630 Loc.isInvalid())
1631 return getOrCreateDefaultLocation(Flags).getPointer();
1632
1633 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1633, __PRETTY_FUNCTION__))
;
1634
1635 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1636 Address LocValue = Address::invalid();
1637 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1638 if (I != OpenMPLocThreadIDMap.end())
1639 LocValue = Address(I->second.DebugLoc, Align);
1640
1641 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1642 // GetOpenMPThreadID was called before this routine.
1643 if (!LocValue.isValid()) {
1644 // Generate "ident_t .kmpc_loc.addr;"
1645 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1646 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1647 Elem.second.DebugLoc = AI.getPointer();
1648 LocValue = AI;
1649
1650 if (!Elem.second.ServiceInsertPt)
1651 setLocThreadIdInsertPt(CGF);
1652 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1653 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1654 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1655 CGF.getTypeSize(IdentQTy));
1656 }
1657
1658 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1659 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1660 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1661 LValue PSource =
1662 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1663
1664 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1665 if (OMPDebugLoc == nullptr) {
1666 SmallString<128> Buffer2;
1667 llvm::raw_svector_ostream OS2(Buffer2);
1668 // Build debug location
1669 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1670 OS2 << ";" << PLoc.getFilename() << ";";
1671 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1672 OS2 << FD->getQualifiedNameAsString();
1673 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1674 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1675 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1676 }
1677 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1678 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1679
1680 // Our callers always pass this to a runtime function, so for
1681 // convenience, go ahead and return a naked pointer.
1682 return LocValue.getPointer();
1683}
1684
1685llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1686 SourceLocation Loc) {
1687 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1687, __PRETTY_FUNCTION__))
;
1688
1689 llvm::Value *ThreadID = nullptr;
1690 // Check whether we've already cached a load of the thread id in this
1691 // function.
1692 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1693 if (I != OpenMPLocThreadIDMap.end()) {
1694 ThreadID = I->second.ThreadID;
1695 if (ThreadID != nullptr)
1696 return ThreadID;
1697 }
1698 // If exceptions are enabled, do not use parameter to avoid possible crash.
1699 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1700 !CGF.getLangOpts().CXXExceptions ||
1701 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1702 if (auto *OMPRegionInfo =
1703 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1704 if (OMPRegionInfo->getThreadIDVariable()) {
1705 // Check if this an outlined function with thread id passed as argument.
1706 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1707 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1708 // If value loaded in entry block, cache it and use it everywhere in
1709 // function.
1710 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1711 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1712 Elem.second.ThreadID = ThreadID;
1713 }
1714 return ThreadID;
1715 }
1716 }
1717 }
1718
1719 // This is not an outlined function region - need to call __kmpc_int32
1720 // kmpc_global_thread_num(ident_t *loc).
1721 // Generate thread id value and cache this value for use across the
1722 // function.
1723 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1724 if (!Elem.second.ServiceInsertPt)
1725 setLocThreadIdInsertPt(CGF);
1726 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1727 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1728 llvm::CallInst *Call = CGF.Builder.CreateCall(
1729 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1730 emitUpdateLocation(CGF, Loc));
1731 Call->setCallingConv(CGF.getRuntimeCC());
1732 Elem.second.ThreadID = Call;
1733 return Call;
1734}
1735
1736void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1737 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1737, __PRETTY_FUNCTION__))
;
1738 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1739 clearLocThreadIdInsertPt(CGF);
1740 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1741 }
1742 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1743 for(auto *D : FunctionUDRMap[CGF.CurFn])
1744 UDRMap.erase(D);
1745 FunctionUDRMap.erase(CGF.CurFn);
1746 }
1747 auto I = FunctionUDMMap.find(CGF.CurFn);
1748 if (I != FunctionUDMMap.end()) {
1749 for(auto *D : I->second)
1750 UDMMap.erase(D);
1751 FunctionUDMMap.erase(I);
1752 }
1753}
1754
1755llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1756 return IdentTy->getPointerTo();
1757}
1758
1759llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1760 if (!Kmpc_MicroTy) {
1761 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1762 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1763 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1764 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1765 }
1766 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1767}
1768
1769llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1770 llvm::FunctionCallee RTLFn = nullptr;
1771 switch (static_cast<OpenMPRTLFunction>(Function)) {
1772 case OMPRTL__kmpc_fork_call: {
1773 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1774 // microtask, ...);
1775 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1776 getKmpc_MicroPointerTy()};
1777 auto *FnTy =
1778 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1779 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1780 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1781 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1782 llvm::LLVMContext &Ctx = F->getContext();
1783 llvm::MDBuilder MDB(Ctx);
1784 // Annotate the callback behavior of the __kmpc_fork_call:
1785 // - The callback callee is argument number 2 (microtask).
1786 // - The first two arguments of the callback callee are unknown (-1).
1787 // - All variadic arguments to the __kmpc_fork_call are passed to the
1788 // callback callee.
1789 F->addMetadata(
1790 llvm::LLVMContext::MD_callback,
1791 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1792 2, {-1, -1},
1793 /* VarArgsArePassed */ true)}));
1794 }
1795 }
1796 break;
1797 }
1798 case OMPRTL__kmpc_global_thread_num: {
1799 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1800 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1801 auto *FnTy =
1802 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1803 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1804 break;
1805 }
1806 case OMPRTL__kmpc_threadprivate_cached: {
1807 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1808 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1809 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1810 CGM.VoidPtrTy, CGM.SizeTy,
1811 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1812 auto *FnTy =
1813 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1814 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1815 break;
1816 }
1817 case OMPRTL__kmpc_critical: {
1818 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1819 // kmp_critical_name *crit);
1820 llvm::Type *TypeParams[] = {
1821 getIdentTyPointerTy(), CGM.Int32Ty,
1822 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1823 auto *FnTy =
1824 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1825 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1826 break;
1827 }
1828 case OMPRTL__kmpc_critical_with_hint: {
1829 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1830 // kmp_critical_name *crit, uintptr_t hint);
1831 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1832 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1833 CGM.IntPtrTy};
1834 auto *FnTy =
1835 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1836 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1837 break;
1838 }
1839 case OMPRTL__kmpc_threadprivate_register: {
1840 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1841 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1842 // typedef void *(*kmpc_ctor)(void *);
1843 auto *KmpcCtorTy =
1844 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1845 /*isVarArg*/ false)->getPointerTo();
1846 // typedef void *(*kmpc_cctor)(void *, void *);
1847 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1848 auto *KmpcCopyCtorTy =
1849 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1850 /*isVarArg*/ false)
1851 ->getPointerTo();
1852 // typedef void (*kmpc_dtor)(void *);
1853 auto *KmpcDtorTy =
1854 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1855 ->getPointerTo();
1856 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1857 KmpcCopyCtorTy, KmpcDtorTy};
1858 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1859 /*isVarArg*/ false);
1860 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1861 break;
1862 }
1863 case OMPRTL__kmpc_end_critical: {
1864 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1865 // kmp_critical_name *crit);
1866 llvm::Type *TypeParams[] = {
1867 getIdentTyPointerTy(), CGM.Int32Ty,
1868 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1869 auto *FnTy =
1870 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1871 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1872 break;
1873 }
1874 case OMPRTL__kmpc_cancel_barrier: {
1875 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1876 // global_tid);
1877 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1878 auto *FnTy =
1879 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1880 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1881 break;
1882 }
1883 case OMPRTL__kmpc_barrier: {
1884 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1885 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1886 auto *FnTy =
1887 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1888 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1889 break;
1890 }
1891 case OMPRTL__kmpc_for_static_fini: {
1892 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1893 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1894 auto *FnTy =
1895 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1896 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1897 break;
1898 }
1899 case OMPRTL__kmpc_push_num_threads: {
1900 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1901 // kmp_int32 num_threads)
1902 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1903 CGM.Int32Ty};
1904 auto *FnTy =
1905 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1906 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1907 break;
1908 }
1909 case OMPRTL__kmpc_serialized_parallel: {
1910 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1911 // global_tid);
1912 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1913 auto *FnTy =
1914 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1915 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1916 break;
1917 }
1918 case OMPRTL__kmpc_end_serialized_parallel: {
1919 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1920 // global_tid);
1921 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1922 auto *FnTy =
1923 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1924 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1925 break;
1926 }
1927 case OMPRTL__kmpc_flush: {
1928 // Build void __kmpc_flush(ident_t *loc);
1929 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1930 auto *FnTy =
1931 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1932 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1933 break;
1934 }
1935 case OMPRTL__kmpc_master: {
1936 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1937 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1938 auto *FnTy =
1939 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1940 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1941 break;
1942 }
1943 case OMPRTL__kmpc_end_master: {
1944 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1945 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1946 auto *FnTy =
1947 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1948 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1949 break;
1950 }
1951 case OMPRTL__kmpc_omp_taskyield: {
1952 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1953 // int end_part);
1954 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1955 auto *FnTy =
1956 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1957 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1958 break;
1959 }
1960 case OMPRTL__kmpc_single: {
1961 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1962 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1963 auto *FnTy =
1964 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1965 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1966 break;
1967 }
1968 case OMPRTL__kmpc_end_single: {
1969 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1970 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1971 auto *FnTy =
1972 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1973 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1974 break;
1975 }
1976 case OMPRTL__kmpc_omp_task_alloc: {
1977 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1978 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1979 // kmp_routine_entry_t *task_entry);
1980 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1981, __PRETTY_FUNCTION__))
1981 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1981, __PRETTY_FUNCTION__))
;
1982 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1983 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1984 // Return void * and then cast to particular kmp_task_t type.
1985 auto *FnTy =
1986 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1987 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1988 break;
1989 }
1990 case OMPRTL__kmpc_omp_target_task_alloc: {
1991 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
1992 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1993 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
1994 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1995, __PRETTY_FUNCTION__))
1995 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1995, __PRETTY_FUNCTION__))
;
1996 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1997 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
1998 CGM.Int64Ty};
1999 // Return void * and then cast to particular kmp_task_t type.
2000 auto *FnTy =
2001 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2002 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2003 break;
2004 }
2005 case OMPRTL__kmpc_omp_task: {
2006 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2007 // *new_task);
2008 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2009 CGM.VoidPtrTy};
2010 auto *FnTy =
2011 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2012 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2013 break;
2014 }
2015 case OMPRTL__kmpc_copyprivate: {
2016 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2017 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2018 // kmp_int32 didit);
2019 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2020 auto *CpyFnTy =
2021 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2022 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2023 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2024 CGM.Int32Ty};
2025 auto *FnTy =
2026 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2027 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2028 break;
2029 }
2030 case OMPRTL__kmpc_reduce: {
2031 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2032 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2033 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2034 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2035 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2036 /*isVarArg=*/false);
2037 llvm::Type *TypeParams[] = {
2038 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2039 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2040 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2041 auto *FnTy =
2042 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2043 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2044 break;
2045 }
2046 case OMPRTL__kmpc_reduce_nowait: {
2047 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2048 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2049 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2050 // *lck);
2051 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2052 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2053 /*isVarArg=*/false);
2054 llvm::Type *TypeParams[] = {
2055 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2056 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2057 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2058 auto *FnTy =
2059 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2060 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2061 break;
2062 }
2063 case OMPRTL__kmpc_end_reduce: {
2064 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2065 // kmp_critical_name *lck);
2066 llvm::Type *TypeParams[] = {
2067 getIdentTyPointerTy(), CGM.Int32Ty,
2068 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2069 auto *FnTy =
2070 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2071 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2072 break;
2073 }
2074 case OMPRTL__kmpc_end_reduce_nowait: {
2075 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2076 // kmp_critical_name *lck);
2077 llvm::Type *TypeParams[] = {
2078 getIdentTyPointerTy(), CGM.Int32Ty,
2079 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2080 auto *FnTy =
2081 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2082 RTLFn =
2083 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2084 break;
2085 }
2086 case OMPRTL__kmpc_omp_task_begin_if0: {
2087 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2088 // *new_task);
2089 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2090 CGM.VoidPtrTy};
2091 auto *FnTy =
2092 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2093 RTLFn =
2094 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2095 break;
2096 }
2097 case OMPRTL__kmpc_omp_task_complete_if0: {
2098 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2099 // *new_task);
2100 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2101 CGM.VoidPtrTy};
2102 auto *FnTy =
2103 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2104 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2105 /*Name=*/"__kmpc_omp_task_complete_if0");
2106 break;
2107 }
2108 case OMPRTL__kmpc_ordered: {
2109 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2110 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2111 auto *FnTy =
2112 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2113 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2114 break;
2115 }
2116 case OMPRTL__kmpc_end_ordered: {
2117 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2118 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2119 auto *FnTy =
2120 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2121 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2122 break;
2123 }
2124 case OMPRTL__kmpc_omp_taskwait: {
2125 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2126 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2127 auto *FnTy =
2128 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2129 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2130 break;
2131 }
2132 case OMPRTL__kmpc_taskgroup: {
2133 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2134 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2135 auto *FnTy =
2136 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2137 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2138 break;
2139 }
2140 case OMPRTL__kmpc_end_taskgroup: {
2141 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2142 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2143 auto *FnTy =
2144 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2145 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2146 break;
2147 }
2148 case OMPRTL__kmpc_push_proc_bind: {
2149 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2150 // int proc_bind)
2151 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2152 auto *FnTy =
2153 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2154 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2155 break;
2156 }
2157 case OMPRTL__kmpc_omp_task_with_deps: {
2158 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2159 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2160 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2161 llvm::Type *TypeParams[] = {
2162 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2163 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2164 auto *FnTy =
2165 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2166 RTLFn =
2167 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2168 break;
2169 }
2170 case OMPRTL__kmpc_omp_wait_deps: {
2171 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2172 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2173 // kmp_depend_info_t *noalias_dep_list);
2174 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2175 CGM.Int32Ty, CGM.VoidPtrTy,
2176 CGM.Int32Ty, CGM.VoidPtrTy};
2177 auto *FnTy =
2178 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2179 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2180 break;
2181 }
2182 case OMPRTL__kmpc_cancellationpoint: {
2183 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2184 // global_tid, kmp_int32 cncl_kind)
2185 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2186 auto *FnTy =
2187 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2188 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2189 break;
2190 }
2191 case OMPRTL__kmpc_cancel: {
2192 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2193 // kmp_int32 cncl_kind)
2194 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2195 auto *FnTy =
2196 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2197 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2198 break;
2199 }
2200 case OMPRTL__kmpc_push_num_teams: {
2201 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2202 // kmp_int32 num_teams, kmp_int32 num_threads)
2203 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2204 CGM.Int32Ty};
2205 auto *FnTy =
2206 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2207 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2208 break;
2209 }
2210 case OMPRTL__kmpc_fork_teams: {
2211 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2212 // microtask, ...);
2213 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2214 getKmpc_MicroPointerTy()};
2215 auto *FnTy =
2216 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2217 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2218 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2219 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2220 llvm::LLVMContext &Ctx = F->getContext();
2221 llvm::MDBuilder MDB(Ctx);
2222 // Annotate the callback behavior of the __kmpc_fork_teams:
2223 // - The callback callee is argument number 2 (microtask).
2224 // - The first two arguments of the callback callee are unknown (-1).
2225 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2226 // callback callee.
2227 F->addMetadata(
2228 llvm::LLVMContext::MD_callback,
2229 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2230 2, {-1, -1},
2231 /* VarArgsArePassed */ true)}));
2232 }
2233 }
2234 break;
2235 }
2236 case OMPRTL__kmpc_taskloop: {
2237 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2238 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2239 // sched, kmp_uint64 grainsize, void *task_dup);
2240 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2241 CGM.IntTy,
2242 CGM.VoidPtrTy,
2243 CGM.IntTy,
2244 CGM.Int64Ty->getPointerTo(),
2245 CGM.Int64Ty->getPointerTo(),
2246 CGM.Int64Ty,
2247 CGM.IntTy,
2248 CGM.IntTy,
2249 CGM.Int64Ty,
2250 CGM.VoidPtrTy};
2251 auto *FnTy =
2252 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2253 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2254 break;
2255 }
2256 case OMPRTL__kmpc_doacross_init: {
2257 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2258 // num_dims, struct kmp_dim *dims);
2259 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2260 CGM.Int32Ty,
2261 CGM.Int32Ty,
2262 CGM.VoidPtrTy};
2263 auto *FnTy =
2264 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2265 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2266 break;
2267 }
2268 case OMPRTL__kmpc_doacross_fini: {
2269 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2270 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2271 auto *FnTy =
2272 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2273 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2274 break;
2275 }
2276 case OMPRTL__kmpc_doacross_post: {
2277 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2278 // *vec);
2279 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2280 CGM.Int64Ty->getPointerTo()};
2281 auto *FnTy =
2282 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2283 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2284 break;
2285 }
2286 case OMPRTL__kmpc_doacross_wait: {
2287 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2288 // *vec);
2289 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2290 CGM.Int64Ty->getPointerTo()};
2291 auto *FnTy =
2292 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2293 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2294 break;
2295 }
2296 case OMPRTL__kmpc_task_reduction_init: {
2297 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2298 // *data);
2299 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2300 auto *FnTy =
2301 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2302 RTLFn =
2303 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2304 break;
2305 }
2306 case OMPRTL__kmpc_task_reduction_get_th_data: {
2307 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2308 // *d);
2309 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2310 auto *FnTy =
2311 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2312 RTLFn = CGM.CreateRuntimeFunction(
2313 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2314 break;
2315 }
2316 case OMPRTL__kmpc_alloc: {
2317 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2318 // al); omp_allocator_handle_t type is void *.
2319 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2320 auto *FnTy =
2321 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2322 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2323 break;
2324 }
2325 case OMPRTL__kmpc_free: {
2326 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2327 // al); omp_allocator_handle_t type is void *.
2328 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2329 auto *FnTy =
2330 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2331 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2332 break;
2333 }
2334 case OMPRTL__kmpc_push_target_tripcount: {
2335 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2336 // size);
2337 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2338 llvm::FunctionType *FnTy =
2339 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2340 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2341 break;
2342 }
2343 case OMPRTL__tgt_target: {
2344 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2345 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2346 // *arg_types);
2347 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2348 CGM.VoidPtrTy,
2349 CGM.Int32Ty,
2350 CGM.VoidPtrPtrTy,
2351 CGM.VoidPtrPtrTy,
2352 CGM.Int64Ty->getPointerTo(),
2353 CGM.Int64Ty->getPointerTo()};
2354 auto *FnTy =
2355 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2356 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2357 break;
2358 }
2359 case OMPRTL__tgt_target_nowait: {
2360 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2361 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2362 // int64_t *arg_types);
2363 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2364 CGM.VoidPtrTy,
2365 CGM.Int32Ty,
2366 CGM.VoidPtrPtrTy,
2367 CGM.VoidPtrPtrTy,
2368 CGM.Int64Ty->getPointerTo(),
2369 CGM.Int64Ty->getPointerTo()};
2370 auto *FnTy =
2371 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2372 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2373 break;
2374 }
2375 case OMPRTL__tgt_target_teams: {
2376 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2377 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2378 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2379 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2380 CGM.VoidPtrTy,
2381 CGM.Int32Ty,
2382 CGM.VoidPtrPtrTy,
2383 CGM.VoidPtrPtrTy,
2384 CGM.Int64Ty->getPointerTo(),
2385 CGM.Int64Ty->getPointerTo(),
2386 CGM.Int32Ty,
2387 CGM.Int32Ty};
2388 auto *FnTy =
2389 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2390 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2391 break;
2392 }
2393 case OMPRTL__tgt_target_teams_nowait: {
2394 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2395 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2396 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2397 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2398 CGM.VoidPtrTy,
2399 CGM.Int32Ty,
2400 CGM.VoidPtrPtrTy,
2401 CGM.VoidPtrPtrTy,
2402 CGM.Int64Ty->getPointerTo(),
2403 CGM.Int64Ty->getPointerTo(),
2404 CGM.Int32Ty,
2405 CGM.Int32Ty};
2406 auto *FnTy =
2407 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2408 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2409 break;
2410 }
2411 case OMPRTL__tgt_register_requires: {
2412 // Build void __tgt_register_requires(int64_t flags);
2413 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2414 auto *FnTy =
2415 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2416 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2417 break;
2418 }
2419 case OMPRTL__tgt_register_lib: {
2420 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2421 QualType ParamTy =
2422 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2423 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2424 auto *FnTy =
2425 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2426 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2427 break;
2428 }
2429 case OMPRTL__tgt_unregister_lib: {
2430 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2431 QualType ParamTy =
2432 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2433 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2434 auto *FnTy =
2435 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2436 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2437 break;
2438 }
2439 case OMPRTL__tgt_target_data_begin: {
2440 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2441 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2442 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2443 CGM.Int32Ty,
2444 CGM.VoidPtrPtrTy,
2445 CGM.VoidPtrPtrTy,
2446 CGM.Int64Ty->getPointerTo(),
2447 CGM.Int64Ty->getPointerTo()};
2448 auto *FnTy =
2449 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2450 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2451 break;
2452 }
2453 case OMPRTL__tgt_target_data_begin_nowait: {
2454 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2455 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2456 // *arg_types);
2457 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2458 CGM.Int32Ty,
2459 CGM.VoidPtrPtrTy,
2460 CGM.VoidPtrPtrTy,
2461 CGM.Int64Ty->getPointerTo(),
2462 CGM.Int64Ty->getPointerTo()};
2463 auto *FnTy =
2464 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2465 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2466 break;
2467 }
2468 case OMPRTL__tgt_target_data_end: {
2469 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2470 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2471 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2472 CGM.Int32Ty,
2473 CGM.VoidPtrPtrTy,
2474 CGM.VoidPtrPtrTy,
2475 CGM.Int64Ty->getPointerTo(),
2476 CGM.Int64Ty->getPointerTo()};
2477 auto *FnTy =
2478 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2479 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2480 break;
2481 }
2482 case OMPRTL__tgt_target_data_end_nowait: {
2483 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2484 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2485 // *arg_types);
2486 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2487 CGM.Int32Ty,
2488 CGM.VoidPtrPtrTy,
2489 CGM.VoidPtrPtrTy,
2490 CGM.Int64Ty->getPointerTo(),
2491 CGM.Int64Ty->getPointerTo()};
2492 auto *FnTy =
2493 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2494 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2495 break;
2496 }
2497 case OMPRTL__tgt_target_data_update: {
2498 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2499 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2500 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2501 CGM.Int32Ty,
2502 CGM.VoidPtrPtrTy,
2503 CGM.VoidPtrPtrTy,
2504 CGM.Int64Ty->getPointerTo(),
2505 CGM.Int64Ty->getPointerTo()};
2506 auto *FnTy =
2507 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2508 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2509 break;
2510 }
2511 case OMPRTL__tgt_target_data_update_nowait: {
2512 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2513 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2514 // *arg_types);
2515 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2516 CGM.Int32Ty,
2517 CGM.VoidPtrPtrTy,
2518 CGM.VoidPtrPtrTy,
2519 CGM.Int64Ty->getPointerTo(),
2520 CGM.Int64Ty->getPointerTo()};
2521 auto *FnTy =
2522 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2523 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2524 break;
2525 }
2526 case OMPRTL__tgt_mapper_num_components: {
2527 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2528 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2529 auto *FnTy =
2530 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2531 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2532 break;
2533 }
2534 case OMPRTL__tgt_push_mapper_component: {
2535 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2536 // *base, void *begin, int64_t size, int64_t type);
2537 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2538 CGM.Int64Ty, CGM.Int64Ty};
2539 auto *FnTy =
2540 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2541 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2542 break;
2543 }
2544 }
2545 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2545, __PRETTY_FUNCTION__))
;
2546 return RTLFn;
2547}
2548
2549llvm::FunctionCallee
2550CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2551 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2552, __PRETTY_FUNCTION__))
2552 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2552, __PRETTY_FUNCTION__))
;
2553 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2554 : "__kmpc_for_static_init_4u")
2555 : (IVSigned ? "__kmpc_for_static_init_8"
2556 : "__kmpc_for_static_init_8u");
2557 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2558 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2559 llvm::Type *TypeParams[] = {
2560 getIdentTyPointerTy(), // loc
2561 CGM.Int32Ty, // tid
2562 CGM.Int32Ty, // schedtype
2563 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2564 PtrTy, // p_lower
2565 PtrTy, // p_upper
2566 PtrTy, // p_stride
2567 ITy, // incr
2568 ITy // chunk
2569 };
2570 auto *FnTy =
2571 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2572 return CGM.CreateRuntimeFunction(FnTy, Name);
2573}
2574
2575llvm::FunctionCallee
2576CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2577 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2578, __PRETTY_FUNCTION__))
2578 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2578, __PRETTY_FUNCTION__))
;
2579 StringRef Name =
2580 IVSize == 32
2581 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2582 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2583 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2584 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2585 CGM.Int32Ty, // tid
2586 CGM.Int32Ty, // schedtype
2587 ITy, // lower
2588 ITy, // upper
2589 ITy, // stride
2590 ITy // chunk
2591 };
2592 auto *FnTy =
2593 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2594 return CGM.CreateRuntimeFunction(FnTy, Name);
2595}
2596
2597llvm::FunctionCallee
2598CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2599 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2600, __PRETTY_FUNCTION__))
2600 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2600, __PRETTY_FUNCTION__))
;
2601 StringRef Name =
2602 IVSize == 32
2603 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2604 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2605 llvm::Type *TypeParams[] = {
2606 getIdentTyPointerTy(), // loc
2607 CGM.Int32Ty, // tid
2608 };
2609 auto *FnTy =
2610 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2611 return CGM.CreateRuntimeFunction(FnTy, Name);
2612}
2613
2614llvm::FunctionCallee
2615CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2616 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2617, __PRETTY_FUNCTION__))
2617 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2617, __PRETTY_FUNCTION__))
;
2618 StringRef Name =
2619 IVSize == 32
2620 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2621 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2622 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2623 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2624 llvm::Type *TypeParams[] = {
2625 getIdentTyPointerTy(), // loc
2626 CGM.Int32Ty, // tid
2627 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2628 PtrTy, // p_lower
2629 PtrTy, // p_upper
2630 PtrTy // p_stride
2631 };
2632 auto *FnTy =
2633 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2634 return CGM.CreateRuntimeFunction(FnTy, Name);
2635}
2636
2637/// Obtain information that uniquely identifies a target entry. This
2638/// consists of the file and device IDs as well as line number associated with
2639/// the relevant entry source location.
2640static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2641 unsigned &DeviceID, unsigned &FileID,
2642 unsigned &LineNum) {
2643 SourceManager &SM = C.getSourceManager();
2644
2645 // The loc should be always valid and have a file ID (the user cannot use
2646 // #pragma directives in macros)
2647
2648 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2648, __PRETTY_FUNCTION__))
;
2649
2650 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2651 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2651, __PRETTY_FUNCTION__))
;
2652
2653 llvm::sys::fs::UniqueID ID;
2654 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2655 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2656 << PLoc.getFilename() << EC.message();
2657
2658 DeviceID = ID.getDevice();
2659 FileID = ID.getFile();
2660 LineNum = PLoc.getLine();
2661}
2662
2663Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2664 if (CGM.getLangOpts().OpenMPSimd)
2665 return Address::invalid();
2666 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2667 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2668 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2669 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2670 HasRequiresUnifiedSharedMemory))) {
2671 SmallString<64> PtrName;
2672 {
2673 llvm::raw_svector_ostream OS(PtrName);
2674 OS << CGM.getMangledName(GlobalDecl(VD));
2675 if (!VD->isExternallyVisible()) {
2676 unsigned DeviceID, FileID, Line;
2677 getTargetEntryUniqueInfo(CGM.getContext(),
2678 VD->getCanonicalDecl()->getBeginLoc(),
2679 DeviceID, FileID, Line);
2680 OS << llvm::format("_%x", FileID);
2681 }
2682 OS << "_decl_tgt_ref_ptr";
2683 }
2684 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2685 if (!Ptr) {
2686 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2687 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2688 PtrName);
2689
2690 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2691 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2692
2693 if (!CGM.getLangOpts().OpenMPIsDevice)
2694 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2695 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2696 }
2697 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2698 }
2699 return Address::invalid();
2700}
2701
2702llvm::Constant *
2703CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2704 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2705, __PRETTY_FUNCTION__))
2705 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2705, __PRETTY_FUNCTION__))
;
2706 // Lookup the entry, lazily creating it if necessary.
2707 std::string Suffix = getName({"cache", ""});
2708 return getOrCreateInternalVariable(
2709 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2710}
2711
2712Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2713 const VarDecl *VD,
2714 Address VDAddr,
2715 SourceLocation Loc) {
2716 if (CGM.getLangOpts().OpenMPUseTLS &&
2717 CGM.getContext().getTargetInfo().isTLSSupported())
2718 return VDAddr;
2719
2720 llvm::Type *VarTy = VDAddr.getElementType();
2721 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2722 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2723 CGM.Int8PtrTy),
2724 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2725 getOrCreateThreadPrivateCache(VD)};
2726 return Address(CGF.EmitRuntimeCall(
2727 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2728 VDAddr.getAlignment());
2729}
2730
2731void CGOpenMPRuntime::emitThreadPrivateVarInit(
2732 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2733 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2734 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2735 // library.
2736 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2737 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2738 OMPLoc);
2739 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2740 // to register constructor/destructor for variable.
2741 llvm::Value *Args[] = {
2742 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2743 Ctor, CopyCtor, Dtor};
2744 CGF.EmitRuntimeCall(
2745 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2746}
2747
2748llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2749 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2750 bool PerformInit, CodeGenFunction *CGF) {
2751 if (CGM.getLangOpts().OpenMPUseTLS &&
2752 CGM.getContext().getTargetInfo().isTLSSupported())
2753 return nullptr;
2754
2755 VD = VD->getDefinition(CGM.getContext());
2756 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2757 QualType ASTTy = VD->getType();
2758
2759 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2760 const Expr *Init = VD->getAnyInitializer();
2761 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2762 // Generate function that re-emits the declaration's initializer into the
2763 // threadprivate copy of the variable VD
2764 CodeGenFunction CtorCGF(CGM);
2765 FunctionArgList Args;
2766 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2767 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2768 ImplicitParamDecl::Other);
2769 Args.push_back(&Dst);
2770
2771 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2772 CGM.getContext().VoidPtrTy, Args);
2773 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2774 std::string Name = getName({"__kmpc_global_ctor_", ""});
2775 llvm::Function *Fn =
2776 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2777 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2778 Args, Loc, Loc);
2779 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2780 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2781 CGM.getContext().VoidPtrTy, Dst.getLocation());
2782 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2783 Arg = CtorCGF.Builder.CreateElementBitCast(
2784 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2785 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2786 /*IsInitializer=*/true);
2787 ArgVal = CtorCGF.EmitLoadOfScalar(
2788 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2789 CGM.getContext().VoidPtrTy, Dst.getLocation());
2790 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2791 CtorCGF.FinishFunction();
2792 Ctor = Fn;
2793 }
2794 if (VD->getType().isDestructedType() != QualType::DK_none) {
2795 // Generate function that emits destructor call for the threadprivate copy
2796 // of the variable VD
2797 CodeGenFunction DtorCGF(CGM);
2798 FunctionArgList Args;
2799 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2800 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2801 ImplicitParamDecl::Other);
2802 Args.push_back(&Dst);
2803
2804 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2805 CGM.getContext().VoidTy, Args);
2806 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2807 std::string Name = getName({"__kmpc_global_dtor_", ""});
2808 llvm::Function *Fn =
2809 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2810 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2811 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2812 Loc, Loc);
2813 // Create a scope with an artificial location for the body of this function.
2814 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2815 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2816 DtorCGF.GetAddrOfLocalVar(&Dst),
2817 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2818 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2819 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2820 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2821 DtorCGF.FinishFunction();
2822 Dtor = Fn;
2823 }
2824 // Do not emit init function if it is not required.
2825 if (!Ctor && !Dtor)
2826 return nullptr;
2827
2828 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2829 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2830 /*isVarArg=*/false)
2831 ->getPointerTo();
2832 // Copying constructor for the threadprivate variable.
2833 // Must be NULL - reserved by runtime, but currently it requires that this
2834 // parameter is always NULL. Otherwise it fires assertion.
2835 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2836 if (Ctor == nullptr) {
2837 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2838 /*isVarArg=*/false)
2839 ->getPointerTo();
2840 Ctor = llvm::Constant::getNullValue(CtorTy);
2841 }
2842 if (Dtor == nullptr) {
2843 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2844 /*isVarArg=*/false)
2845 ->getPointerTo();
2846 Dtor = llvm::Constant::getNullValue(DtorTy);
2847 }
2848 if (!CGF) {
2849 auto *InitFunctionTy =
2850 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2851 std::string Name = getName({"__omp_threadprivate_init_", ""});
2852 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2853 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2854 CodeGenFunction InitCGF(CGM);
2855 FunctionArgList ArgList;
2856 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2857 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2858 Loc, Loc);
2859 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2860 InitCGF.FinishFunction();
2861 return InitFunction;
2862 }
2863 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2864 }
2865 return nullptr;
2866}
2867
2868bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2869 llvm::GlobalVariable *Addr,
2870 bool PerformInit) {
2871 if (CGM.getLangOpts().OMPTargetTriples.empty())
1
Assuming the condition is false
2
Taking false branch
2872 return false;
2873 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2874 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2875 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
3
Calling 'Optional::operator bool'
11
Returning from 'Optional::operator bool'
12
Assuming the condition is false
2876 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
13
Assuming the condition is false
2877 HasRequiresUnifiedSharedMemory))
2878 return CGM.getLangOpts().OpenMPIsDevice;
2879 VD = VD->getDefinition(CGM.getContext());
14
Value assigned to 'VD'
2880 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
15
Assuming 'VD' is null
16
Taking false branch
2881 return CGM.getLangOpts().OpenMPIsDevice;
2882
2883 QualType ASTTy = VD->getType();
17
Called C++ object pointer is null
2884
2885 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2886 // Produce the unique prefix to identify the new target regions. We use
2887 // the source location of the variable declaration which we know to not
2888 // conflict with any target region.
2889 unsigned DeviceID;
2890 unsigned FileID;
2891 unsigned Line;
2892 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2893 SmallString<128> Buffer, Out;
2894 {
2895 llvm::raw_svector_ostream OS(Buffer);
2896 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2897 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2898 }
2899
2900 const Expr *Init = VD->getAnyInitializer();
2901 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2902 llvm::Constant *Ctor;
2903 llvm::Constant *ID;
2904 if (CGM.getLangOpts().OpenMPIsDevice) {
2905 // Generate function that re-emits the declaration's initializer into
2906 // the threadprivate copy of the variable VD
2907 CodeGenFunction CtorCGF(CGM);
2908
2909 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2910 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2911 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2912 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2913 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2914 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2915 FunctionArgList(), Loc, Loc);
2916 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2917 CtorCGF.EmitAnyExprToMem(Init,
2918 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2919 Init->getType().getQualifiers(),
2920 /*IsInitializer=*/true);
2921 CtorCGF.FinishFunction();
2922 Ctor = Fn;
2923 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2924 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2925 } else {
2926 Ctor = new llvm::GlobalVariable(
2927 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2928 llvm::GlobalValue::PrivateLinkage,
2929 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2930 ID = Ctor;
2931 }
2932
2933 // Register the information for the entry associated with the constructor.
2934 Out.clear();
2935 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2936 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2937 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2938 }
2939 if (VD->getType().isDestructedType() != QualType::DK_none) {
2940 llvm::Constant *Dtor;
2941 llvm::Constant *ID;
2942 if (CGM.getLangOpts().OpenMPIsDevice) {
2943 // Generate function that emits destructor call for the threadprivate
2944 // copy of the variable VD
2945 CodeGenFunction DtorCGF(CGM);
2946
2947 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2948 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2949 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2950 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2951 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2952 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2953 FunctionArgList(), Loc, Loc);
2954 // Create a scope with an artificial location for the body of this
2955 // function.
2956 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2957 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2958 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2959 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2960 DtorCGF.FinishFunction();
2961 Dtor = Fn;
2962 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2963 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2964 } else {
2965 Dtor = new llvm::GlobalVariable(
2966 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2967 llvm::GlobalValue::PrivateLinkage,
2968 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2969 ID = Dtor;
2970 }
2971 // Register the information for the entry associated with the destructor.
2972 Out.clear();
2973 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2974 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2975 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2976 }
2977 return CGM.getLangOpts().OpenMPIsDevice;
2978}
2979
2980Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2981 QualType VarType,
2982 StringRef Name) {
2983 std::string Suffix = getName({"artificial", ""});
2984 std::string CacheSuffix = getName({"cache", ""});
2985 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2986 llvm::Value *GAddr =
2987 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2988 llvm::Value *Args[] = {
2989 emitUpdateLocation(CGF, SourceLocation()),
2990 getThreadID(CGF, SourceLocation()),
2991 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2992 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2993 /*isSigned=*/false),
2994 getOrCreateInternalVariable(
2995 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2996 return Address(
2997 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2998 CGF.EmitRuntimeCall(
2999 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3000 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3001 CGM.getPointerAlign());
3002}
3003
3004void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
3005 const RegionCodeGenTy &ThenGen,
3006 const RegionCodeGenTy &ElseGen) {
3007 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3008
3009 // If the condition constant folds and can be elided, try to avoid emitting
3010 // the condition and the dead arm of the if/else.
3011 bool CondConstant;
3012 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
3013 if (CondConstant)
3014 ThenGen(CGF);
3015 else
3016 ElseGen(CGF);
3017 return;
3018 }
3019
3020 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3021 // emit the conditional branch.
3022 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3023 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3024 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3025 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3026
3027 // Emit the 'then' code.
3028 CGF.EmitBlock(ThenBlock);
3029 ThenGen(CGF);
3030 CGF.EmitBranch(ContBlock);
3031 // Emit the 'else' code if present.
3032 // There is no need to emit line number for unconditional branch.
3033 (void)ApplyDebugLocation::CreateEmpty(CGF);
3034 CGF.EmitBlock(ElseBlock);
3035 ElseGen(CGF);
3036 // There is no need to emit line number for unconditional branch.
3037 (void)ApplyDebugLocation::CreateEmpty(CGF);
3038 CGF.EmitBranch(ContBlock);
3039 // Emit the continuation block for code after the if.
3040 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3041}
3042
3043void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3044 llvm::Function *OutlinedFn,
3045 ArrayRef<llvm::Value *> CapturedVars,
3046 const Expr *IfCond) {
3047 if (!CGF.HaveInsertPoint())
3048 return;
3049 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3050 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3051 PrePostActionTy &) {
3052 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3053 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3054 llvm::Value *Args[] = {
3055 RTLoc,
3056 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3057 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3058 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3059 RealArgs.append(std::begin(Args), std::end(Args));
3060 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3061
3062 llvm::FunctionCallee RTLFn =
3063 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3064 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3065 };
3066 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3067 PrePostActionTy &) {
3068 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3069 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3070 // Build calls:
3071 // __kmpc_serialized_parallel(&Loc, GTid);
3072 llvm::Value *Args[] = {RTLoc, ThreadID};
3073 CGF.EmitRuntimeCall(
3074 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3075
3076 // OutlinedFn(&GTid, &zero, CapturedStruct);
3077 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3078 /*Name*/ ".zero.addr");
3079 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
3080 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3081 // ThreadId for serialized parallels is 0.
3082 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3083 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
3084 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3085 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3086
3087 // __kmpc_end_serialized_parallel(&Loc, GTid);
3088 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3089 CGF.EmitRuntimeCall(
3090 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3091 EndArgs);
3092 };
3093 if (IfCond) {
3094 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
3095 } else {
3096 RegionCodeGenTy ThenRCG(ThenGen);
3097 ThenRCG(CGF);
3098 }
3099}
3100
3101// If we're inside an (outlined) parallel region, use the region info's
3102// thread-ID variable (it is passed in a first argument of the outlined function
3103// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3104// regular serial code region, get thread ID by calling kmp_int32
3105// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3106// return the address of that temp.
3107Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3108 SourceLocation Loc) {
3109 if (auto *OMPRegionInfo =
3110 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3111 if (OMPRegionInfo->getThreadIDVariable())
3112 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
3113
3114 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3115 QualType Int32Ty =
3116 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3117 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3118 CGF.EmitStoreOfScalar(ThreadID,
3119 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3120
3121 return ThreadIDTemp;
3122}
3123
3124llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3125 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3126 SmallString<256> Buffer;
3127 llvm::raw_svector_ostream Out(Buffer);
3128 Out << Name;
3129 StringRef RuntimeName = Out.str();
3130 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3131 if (Elem.second) {
3132 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3133, __PRETTY_FUNCTION__))
3133 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3133, __PRETTY_FUNCTION__))
;
3134 return &*Elem.second;
3135 }
3136
3137 return Elem.second = new llvm::GlobalVariable(
3138 CGM.getModule(), Ty, /*IsConstant*/ false,
3139 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3140 Elem.first(), /*InsertBefore=*/nullptr,
3141 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3142}
3143
3144llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3145 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3146 std::string Name = getName({Prefix, "var"});
3147 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3148}
3149
3150namespace {
3151/// Common pre(post)-action for different OpenMP constructs.
3152class CommonActionTy final : public PrePostActionTy {
3153 llvm::FunctionCallee EnterCallee;
3154 ArrayRef<llvm::Value *> EnterArgs;
3155 llvm::FunctionCallee ExitCallee;
3156 ArrayRef<llvm::Value *> ExitArgs;
3157 bool Conditional;
3158 llvm::BasicBlock *ContBlock = nullptr;
3159
3160public:
3161 CommonActionTy(llvm::FunctionCallee EnterCallee,
3162 ArrayRef<llvm::Value *> EnterArgs,
3163 llvm::FunctionCallee ExitCallee,
3164 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3165 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3166 ExitArgs(ExitArgs), Conditional(Conditional) {}
3167 void Enter(CodeGenFunction &CGF) override {
3168 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3169 if (Conditional) {
3170 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3171 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3172 ContBlock = CGF.createBasicBlock("omp_if.end");
3173 // Generate the branch (If-stmt)
3174 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3175 CGF.EmitBlock(ThenBlock);
3176 }
3177 }
3178 void Done(CodeGenFunction &CGF) {
3179 // Emit the rest of blocks/branches
3180 CGF.EmitBranch(ContBlock);
3181 CGF.EmitBlock(ContBlock, true);
3182 }
3183 void Exit(CodeGenFunction &CGF) override {
3184 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3185 }
3186};
3187} // anonymous namespace
3188
3189void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3190 StringRef CriticalName,
3191 const RegionCodeGenTy &CriticalOpGen,
3192 SourceLocation Loc, const Expr *Hint) {
3193 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3194 // CriticalOpGen();
3195 // __kmpc_end_critical(ident_t *, gtid, Lock);
3196 // Prepare arguments and build a call to __kmpc_critical
3197 if (!CGF.HaveInsertPoint())
3198 return;
3199 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3200 getCriticalRegionLock(CriticalName)};
3201 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3202 std::end(Args));
3203 if (Hint) {
3204 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3205 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3206 }
3207 CommonActionTy Action(
3208 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3209 : OMPRTL__kmpc_critical),
3210 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3211 CriticalOpGen.setAction(Action);
3212 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3213}
3214
3215void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3216 const RegionCodeGenTy &MasterOpGen,
3217 SourceLocation Loc) {
3218 if (!CGF.HaveInsertPoint())
3219 return;
3220 // if(__kmpc_master(ident_t *, gtid)) {
3221 // MasterOpGen();
3222 // __kmpc_end_master(ident_t *, gtid);
3223 // }
3224 // Prepare arguments and build a call to __kmpc_master
3225 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3226 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3227 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3228 /*Conditional=*/true);
3229 MasterOpGen.setAction(Action);
3230 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3231 Action.Done(CGF);
3232}
3233
3234void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3235 SourceLocation Loc) {
3236 if (!CGF.HaveInsertPoint())
3237 return;
3238 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3239 llvm::Value *Args[] = {
3240 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3241 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3242 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3243 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3244 Region->emitUntiedSwitch(CGF);
3245}
3246
3247void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3248 const RegionCodeGenTy &TaskgroupOpGen,
3249 SourceLocation Loc) {
3250 if (!CGF.HaveInsertPoint())
3251 return;
3252 // __kmpc_taskgroup(ident_t *, gtid);
3253 // TaskgroupOpGen();
3254 // __kmpc_end_taskgroup(ident_t *, gtid);
3255 // Prepare arguments and build a call to __kmpc_taskgroup
3256 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3257 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3258 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3259 Args);
3260 TaskgroupOpGen.setAction(Action);
3261 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3262}
3263
3264/// Given an array of pointers to variables, project the address of a
3265/// given variable.
3266static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3267 unsigned Index, const VarDecl *Var) {
3268 // Pull out the pointer to the variable.
3269 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3270 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3271
3272 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3273 Addr = CGF.Builder.CreateElementBitCast(
3274 Addr, CGF.ConvertTypeForMem(Var->getType()));
3275 return Addr;
3276}
3277
3278static llvm::Value *emitCopyprivateCopyFunction(
3279 CodeGenModule &CGM, llvm::Type *ArgsType,
3280 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3281 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3282 SourceLocation Loc) {
3283 ASTContext &C = CGM.getContext();
3284 // void copy_func(void *LHSArg, void *RHSArg);
3285 FunctionArgList Args;
3286 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3287 ImplicitParamDecl::Other);
3288 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3289 ImplicitParamDecl::Other);
3290 Args.push_back(&LHSArg);
3291 Args.push_back(&RHSArg);
3292 const auto &CGFI =
3293 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3294 std::string Name =
3295 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3296 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3297 llvm::GlobalValue::InternalLinkage, Name,
3298 &CGM.getModule());
3299 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3300 Fn->setDoesNotRecurse();
3301 CodeGenFunction CGF(CGM);
3302 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3303 // Dest = (void*[n])(LHSArg);
3304 // Src = (void*[n])(RHSArg);
3305 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3306 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3307 ArgsType), CGF.getPointerAlign());
3308 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3309 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3310 ArgsType), CGF.getPointerAlign());
3311 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3312 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3313 // ...
3314 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3315 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3316 const auto *DestVar =
3317 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3318 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3319
3320 const auto *SrcVar =
3321 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3322 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3323
3324 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3325 QualType Type = VD->getType();
3326 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3327 }
3328 CGF.FinishFunction();
3329 return Fn;
3330}
3331
3332void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3333 const RegionCodeGenTy &SingleOpGen,
3334 SourceLocation Loc,
3335 ArrayRef<const Expr *> CopyprivateVars,
3336 ArrayRef<const Expr *> SrcExprs,
3337 ArrayRef<const Expr *> DstExprs,
3338 ArrayRef<const Expr *> AssignmentOps) {
3339 if (!CGF.HaveInsertPoint())
3340 return;
3341 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
3342 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
3343 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3343, __PRETTY_FUNCTION__))
;
3344 ASTContext &C = CGM.getContext();
3345 // int32 did_it = 0;
3346 // if(__kmpc_single(ident_t *, gtid)) {
3347 // SingleOpGen();
3348 // __kmpc_end_single(ident_t *, gtid);
3349 // did_it = 1;
3350 // }
3351 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3352 // <copy_func>, did_it);
3353
3354 Address DidIt = Address::invalid();
3355 if (!CopyprivateVars.empty()) {
3356 // int32 did_it = 0;
3357 QualType KmpInt32Ty =
3358 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3359 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3360 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3361 }
3362 // Prepare arguments and build a call to __kmpc_single
3363 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3364 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3365 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3366 /*Conditional=*/true);
3367 SingleOpGen.setAction(Action);
3368 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3369 if (DidIt.isValid()) {
3370 // did_it = 1;
3371 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3372 }
3373 Action.Done(CGF);
3374 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3375 // <copy_func>, did_it);
3376 if (DidIt.isValid()) {
3377 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3378 QualType CopyprivateArrayTy =
3379 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3380 /*IndexTypeQuals=*/0);
3381 // Create a list of all private variables for copyprivate.
3382 Address CopyprivateList =
3383 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3384 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3385 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3386 CGF.Builder.CreateStore(
3387 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3388 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3389 Elem);
3390 }
3391 // Build function that copies private values from single region to all other
3392 // threads in the corresponding parallel region.
3393 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3394 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3395 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3396 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3397 Address CL =
3398 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3399 CGF.VoidPtrTy);
3400 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3401 llvm::Value *Args[] = {
3402 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3403 getThreadID(CGF, Loc), // i32 <gtid>
3404 BufSize, // size_t <buf_size>
3405 CL.getPointer(), // void *<copyprivate list>
3406 CpyFn, // void (*) (void *, void *) <copy_func>
3407 DidItVal // i32 did_it
3408 };
3409 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3410 }
3411}
3412
3413void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3414 const RegionCodeGenTy &OrderedOpGen,
3415 SourceLocation Loc, bool IsThreads) {
3416 if (!CGF.HaveInsertPoint())
3417 return;
3418 // __kmpc_ordered(ident_t *, gtid);
3419 // OrderedOpGen();
3420 // __kmpc_end_ordered(ident_t *, gtid);
3421 // Prepare arguments and build a call to __kmpc_ordered
3422 if (IsThreads) {
3423 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3424 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3425 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3426 Args);
3427 OrderedOpGen.setAction(Action);
3428 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3429 return;
3430 }
3431 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3432}
3433
3434unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3435 unsigned Flags;
3436 if (Kind == OMPD_for)
3437 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3438 else if (Kind == OMPD_sections)
3439 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3440 else if (Kind == OMPD_single)
3441 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3442 else if (Kind == OMPD_barrier)
3443 Flags = OMP_IDENT_BARRIER_EXPL;
3444 else
3445 Flags = OMP_IDENT_BARRIER_IMPL;
3446 return Flags;
3447}
3448
3449void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3450 CodeGenFunction &CGF, const OMPLoopDirective &S,
3451 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3452 // Check if the loop directive is actually a doacross loop directive. In this
3453 // case choose static, 1 schedule.
3454 if (llvm::any_of(
3455 S.getClausesOfKind<OMPOrderedClause>(),
3456 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3457 ScheduleKind = OMPC_SCHEDULE_static;
3458 // Chunk size is 1 in this case.
3459 llvm::APInt ChunkSize(32, 1);
3460 ChunkExpr = IntegerLiteral::Create(
3461 CGF.getContext(), ChunkSize,
3462 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3463 SourceLocation());
3464 }
3465}
3466
3467void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3468 OpenMPDirectiveKind Kind, bool EmitChecks,
3469 bool ForceSimpleCall) {
3470 if (!CGF.HaveInsertPoint())
3471 return;
3472 // Build call __kmpc_cancel_barrier(loc, thread_id);
3473 // Build call __kmpc_barrier(loc, thread_id);
3474 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3475 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3476 // thread_id);
3477 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3478 getThreadID(CGF, Loc)};
3479 if (auto *OMPRegionInfo =
3480 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3481 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3482 llvm::Value *Result = CGF.EmitRuntimeCall(
3483 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3484 if (EmitChecks) {
3485 // if (__kmpc_cancel_barrier()) {
3486 // exit from construct;
3487 // }
3488 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3489 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3490 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3491 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3492 CGF.EmitBlock(ExitBB);
3493 // exit from construct;
3494 CodeGenFunction::JumpDest CancelDestination =
3495 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3496 CGF.EmitBranchThroughCleanup(CancelDestination);
3497 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3498 }
3499 return;
3500 }
3501 }
3502 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3503}
3504
3505/// Map the OpenMP loop schedule to the runtime enumeration.
3506static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3507 bool Chunked, bool Ordered) {
3508 switch (ScheduleKind) {
3509 case OMPC_SCHEDULE_static:
3510 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3511 : (Ordered ? OMP_ord_static : OMP_sch_static);
3512 case OMPC_SCHEDULE_dynamic:
3513 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3514 case OMPC_SCHEDULE_guided:
3515 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3516 case OMPC_SCHEDULE_runtime:
3517 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3518 case OMPC_SCHEDULE_auto:
3519 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3520 case OMPC_SCHEDULE_unknown:
3521 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3521, __PRETTY_FUNCTION__))
;
3522 return Ordered ? OMP_ord_static : OMP_sch_static;
3523 }
3524 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3524)
;
3525}
3526
3527/// Map the OpenMP distribute schedule to the runtime enumeration.
3528static OpenMPSchedType
3529getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3530 // only static is allowed for dist_schedule
3531 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3532}
3533
3534bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3535 bool Chunked) const {
3536 OpenMPSchedType Schedule =
3537 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3538 return Schedule == OMP_sch_static;
3539}
3540
3541bool CGOpenMPRuntime::isStaticNonchunked(
3542 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3543 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3544 return Schedule == OMP_dist_sch_static;
3545}
3546
3547bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3548 bool Chunked) const {
3549 OpenMPSchedType Schedule =
3550 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3551 return Schedule == OMP_sch_static_chunked;
3552}
3553
3554bool CGOpenMPRuntime::isStaticChunked(
3555 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3556 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3557 return Schedule == OMP_dist_sch_static_chunked;
3558}
3559
3560bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3561 OpenMPSchedType Schedule =
3562 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3563 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3563, __PRETTY_FUNCTION__))
;
3564 return Schedule != OMP_sch_static;
3565}
3566
3567static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3568 OpenMPScheduleClauseModifier M1,
3569 OpenMPScheduleClauseModifier M2) {
3570 int Modifier = 0;
3571 switch (M1) {
3572 case OMPC_SCHEDULE_MODIFIER_monotonic:
3573 Modifier = OMP_sch_modifier_monotonic;
3574 break;
3575 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3576 Modifier = OMP_sch_modifier_nonmonotonic;
3577 break;
3578 case OMPC_SCHEDULE_MODIFIER_simd:
3579 if (Schedule == OMP_sch_static_chunked)
3580 Schedule = OMP_sch_static_balanced_chunked;
3581 break;
3582 case OMPC_SCHEDULE_MODIFIER_last:
3583 case OMPC_SCHEDULE_MODIFIER_unknown:
3584 break;
3585 }
3586 switch (M2) {
3587 case OMPC_SCHEDULE_MODIFIER_monotonic:
3588 Modifier = OMP_sch_modifier_monotonic;
3589 break;
3590 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3591 Modifier = OMP_sch_modifier_nonmonotonic;
3592 break;
3593 case OMPC_SCHEDULE_MODIFIER_simd:
3594 if (Schedule == OMP_sch_static_chunked)
3595 Schedule = OMP_sch_static_balanced_chunked;
3596 break;
3597 case OMPC_SCHEDULE_MODIFIER_last:
3598 case OMPC_SCHEDULE_MODIFIER_unknown:
3599 break;
3600 }
3601 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3602 // If the static schedule kind is specified or if the ordered clause is
3603 // specified, and if the nonmonotonic modifier is not specified, the effect is
3604 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3605 // modifier is specified, the effect is as if the nonmonotonic modifier is
3606 // specified.
3607 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3608 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3609 Schedule == OMP_sch_static_balanced_chunked ||
3610 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static))
3611 Modifier = OMP_sch_modifier_nonmonotonic;
3612 }
3613 return Schedule | Modifier;
3614}
3615
3616void CGOpenMPRuntime::emitForDispatchInit(
3617 CodeGenFunction &CGF, SourceLocation Loc,
3618 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3619 bool Ordered, const DispatchRTInput &DispatchValues) {
3620 if (!CGF.HaveInsertPoint())
3621 return;
3622 OpenMPSchedType Schedule = getRuntimeSchedule(
3623 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3624 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3625 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3626 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
3627 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3627, __PRETTY_FUNCTION__))
;
3628 // Call __kmpc_dispatch_init(
3629 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3630 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3631 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3632
3633 // If the Chunk was not specified in the clause - use default value 1.
3634 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3635 : CGF.Builder.getIntN(IVSize, 1);
3636 llvm::Value *Args[] = {
3637 emitUpdateLocation(CGF, Loc),
3638 getThreadID(CGF, Loc),
3639 CGF.Builder.getInt32(addMonoNonMonoModifier(
3640 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3641 DispatchValues.LB, // Lower
3642 DispatchValues.UB, // Upper
3643 CGF.Builder.getIntN(IVSize, 1), // Stride
3644 Chunk // Chunk
3645 };
3646 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3647}
3648
3649static void emitForStaticInitCall(
3650 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3651 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3652 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3653 const CGOpenMPRuntime::StaticRTInput &Values) {
3654 if (!CGF.HaveInsertPoint())
3655 return;
3656
3657 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3657, __PRETTY_FUNCTION__))
;
3658 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3659 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3660 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3661 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
3662 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3662, __PRETTY_FUNCTION__))
;
3663
3664 // Call __kmpc_for_static_init(
3665 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3666 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3667 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3668 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3669 llvm::Value *Chunk = Values.Chunk;
3670 if (Chunk == nullptr) {
3671 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
3672 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
3673 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3673, __PRETTY_FUNCTION__))
;
3674 // If the Chunk was not specified in the clause - use default value 1.
3675 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3676 } else {
3677 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3678 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3679 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3680 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
3681 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3681, __PRETTY_FUNCTION__))
;
3682 }
3683 llvm::Value *Args[] = {
3684 UpdateLocation,
3685 ThreadId,
3686 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3687 M2)), // Schedule type
3688 Values.IL.getPointer(), // &isLastIter
3689 Values.LB.getPointer(), // &LB
3690 Values.UB.getPointer(), // &UB
3691 Values.ST.getPointer(), // &Stride
3692 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3693 Chunk // Chunk
3694 };
3695 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3696}
3697
3698void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3699 SourceLocation Loc,
3700 OpenMPDirectiveKind DKind,
3701 const OpenMPScheduleTy &ScheduleKind,
3702 const StaticRTInput &Values) {
3703 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3704 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3705 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3706 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3708 isOpenMPLoopDirective(DKind)
3709 ? OMP_IDENT_WORK_LOOP
3710 : OMP_IDENT_WORK_SECTIONS);
3711 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3712 llvm::FunctionCallee StaticInitFunction =
3713 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3714 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3715 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3716}
3717
3718void CGOpenMPRuntime::emitDistributeStaticInit(
3719 CodeGenFunction &CGF, SourceLocation Loc,
3720 OpenMPDistScheduleClauseKind SchedKind,
3721 const CGOpenMPRuntime::StaticRTInput &Values) {
3722 OpenMPSchedType ScheduleNum =
3723 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3724 llvm::Value *UpdatedLocation =
3725 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3726 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3727 llvm::FunctionCallee StaticInitFunction =
3728 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3729 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3730 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3731 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3732}
3733
3734void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3735 SourceLocation Loc,
3736 OpenMPDirectiveKind DKind) {
3737 if (!CGF.HaveInsertPoint())
3738 return;
3739 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3740 llvm::Value *Args[] = {
3741 emitUpdateLocation(CGF, Loc,
3742 isOpenMPDistributeDirective(DKind)
3743 ? OMP_IDENT_WORK_DISTRIBUTE
3744 : isOpenMPLoopDirective(DKind)
3745 ? OMP_IDENT_WORK_LOOP
3746 : OMP_IDENT_WORK_SECTIONS),
3747 getThreadID(CGF, Loc)};
3748 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3749 Args);
3750}
3751
3752void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3753 SourceLocation Loc,
3754 unsigned IVSize,
3755 bool IVSigned) {
3756 if (!CGF.HaveInsertPoint())
3757 return;
3758 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3759 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3760 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3761}
3762
3763llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3764 SourceLocation Loc, unsigned IVSize,
3765 bool IVSigned, Address IL,
3766 Address LB, Address UB,
3767 Address ST) {
3768 // Call __kmpc_dispatch_next(
3769 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3770 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3771 // kmp_int[32|64] *p_stride);
3772 llvm::Value *Args[] = {
3773 emitUpdateLocation(CGF, Loc),
3774 getThreadID(CGF, Loc),
3775 IL.getPointer(), // &isLastIter
3776 LB.getPointer(), // &Lower
3777 UB.getPointer(), // &Upper
3778 ST.getPointer() // &Stride
3779 };
3780 llvm::Value *Call =
3781 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3782 return CGF.EmitScalarConversion(
3783 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3784 CGF.getContext().BoolTy, Loc);
3785}
3786
3787void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3788 llvm::Value *NumThreads,
3789 SourceLocation Loc) {
3790 if (!CGF.HaveInsertPoint())
3791 return;
3792 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3793 llvm::Value *Args[] = {
3794 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3795 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3796 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3797 Args);
3798}
3799
3800void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3801 OpenMPProcBindClauseKind ProcBind,
3802 SourceLocation Loc) {
3803 if (!CGF.HaveInsertPoint())
3804 return;
3805 // Constants for proc bind value accepted by the runtime.
3806 enum ProcBindTy {
3807 ProcBindFalse = 0,
3808 ProcBindTrue,
3809 ProcBindMaster,
3810 ProcBindClose,
3811 ProcBindSpread,
3812 ProcBindIntel,
3813 ProcBindDefault
3814 } RuntimeProcBind;
3815 switch (ProcBind) {
3816 case OMPC_PROC_BIND_master:
3817 RuntimeProcBind = ProcBindMaster;
3818 break;
3819 case OMPC_PROC_BIND_close:
3820 RuntimeProcBind = ProcBindClose;
3821 break;
3822 case OMPC_PROC_BIND_spread:
3823 RuntimeProcBind = ProcBindSpread;
3824 break;
3825 case OMPC_PROC_BIND_unknown:
3826 llvm_unreachable("Unsupported proc_bind value.")::llvm::llvm_unreachable_internal("Unsupported proc_bind value."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3826)
;
3827 }
3828 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3829 llvm::Value *Args[] = {
3830 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3831 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3832 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3833}
3834
3835void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3836 SourceLocation Loc) {
3837 if (!CGF.HaveInsertPoint())
3838 return;
3839 // Build call void __kmpc_flush(ident_t *loc)
3840 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3841 emitUpdateLocation(CGF, Loc));
3842}
3843
3844namespace {
3845/// Indexes of fields for type kmp_task_t.
3846enum KmpTaskTFields {
3847 /// List of shared variables.
3848 KmpTaskTShareds,
3849 /// Task routine.
3850 KmpTaskTRoutine,
3851 /// Partition id for the untied tasks.
3852 KmpTaskTPartId,
3853 /// Function with call of destructors for private variables.
3854 Data1,
3855 /// Task priority.
3856 Data2,
3857 /// (Taskloops only) Lower bound.
3858 KmpTaskTLowerBound,
3859 /// (Taskloops only) Upper bound.
3860 KmpTaskTUpperBound,
3861 /// (Taskloops only) Stride.
3862 KmpTaskTStride,
3863 /// (Taskloops only) Is last iteration flag.
3864 KmpTaskTLastIter,
3865 /// (Taskloops only) Reduction data.
3866 KmpTaskTReductions,
3867};
3868} // anonymous namespace
3869
3870bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3871 return OffloadEntriesTargetRegion.empty() &&
3872 OffloadEntriesDeviceGlobalVar.empty();
3873}
3874
3875/// Initialize target region entry.
3876void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3877 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3878 StringRef ParentName, unsigned LineNum,
3879 unsigned Order) {
3880 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
3881 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
3882 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3882, __PRETTY_FUNCTION__))
;
3883 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3884 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3885 OMPTargetRegionEntryTargetRegion);
3886 ++OffloadingEntriesNum;
3887}
3888
3889void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3890 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3891 StringRef ParentName, unsigned LineNum,
3892 llvm::Constant *Addr, llvm::Constant *ID,
3893 OMPTargetRegionEntryKind Flags) {
3894 // If we are emitting code for a target, the entry is already initialized,
3895 // only has to be registered.
3896 if (CGM.getLangOpts().OpenMPIsDevice) {
3897 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3898 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3899 DiagnosticsEngine::Error,
3900 "Unable to find target region on line '%0' in the device code.");
3901 CGM.getDiags().Report(DiagID) << LineNum;
3902 return;
3903 }
3904 auto &Entry =
3905 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3906 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3906, __PRETTY_FUNCTION__))
;
3907 Entry.setAddress(Addr);
3908 Entry.setID(ID);
3909 Entry.setFlags(Flags);
3910 } else {
3911 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3912 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3913 ++OffloadingEntriesNum;
3914 }
3915}
3916
3917bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3918 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3919 unsigned LineNum) const {
3920 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3921 if (PerDevice == OffloadEntriesTargetRegion.end())
3922 return false;
3923 auto PerFile = PerDevice->second.find(FileID);
3924 if (PerFile == PerDevice->second.end())
3925 return false;
3926 auto PerParentName = PerFile->second.find(ParentName);
3927 if (PerParentName == PerFile->second.end())
3928 return false;
3929 auto PerLine = PerParentName->second.find(LineNum);
3930 if (PerLine == PerParentName->second.end())
3931 return false;
3932 // Fail if this entry is already registered.
3933 if (PerLine->second.getAddress() || PerLine->second.getID())
3934 return false;
3935 return true;
3936}
3937
3938void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3939 const OffloadTargetRegionEntryInfoActTy &Action) {
3940 // Scan all target region entries and perform the provided action.
3941 for (const auto &D : OffloadEntriesTargetRegion)
3942 for (const auto &F : D.second)
3943 for (const auto &P : F.second)
3944 for (const auto &L : P.second)
3945 Action(D.first, F.first, P.first(), L.first, L.second);
3946}
3947
3948void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3949 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3950 OMPTargetGlobalVarEntryKind Flags,
3951 unsigned Order) {
3952 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
3953 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
3954 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3954, __PRETTY_FUNCTION__))
;
3955 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3956 ++OffloadingEntriesNum;
3957}
3958
3959void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3960 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3961 CharUnits VarSize,
3962 OMPTargetGlobalVarEntryKind Flags,
3963 llvm::GlobalValue::LinkageTypes Linkage) {
3964 if (CGM.getLangOpts().OpenMPIsDevice) {
3965 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3966 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3967, __PRETTY_FUNCTION__))
3967 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3967, __PRETTY_FUNCTION__))
;
3968 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
3969 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3969, __PRETTY_FUNCTION__))
;
3970 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3971 if (Entry.getVarSize().isZero()) {
3972 Entry.setVarSize(VarSize);
3973 Entry.setLinkage(Linkage);
3974 }
3975 return;
3976 }
3977 Entry.setVarSize(VarSize);
3978 Entry.setLinkage(Linkage);
3979 Entry.setAddress(Addr);
3980 } else {
3981 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3982 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3983 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3984, __PRETTY_FUNCTION__))
3984 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3984, __PRETTY_FUNCTION__))
;
3985 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
3986 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3986, __PRETTY_FUNCTION__))
;
3987 if (Entry.getVarSize().isZero()) {
3988 Entry.setVarSize(VarSize);
3989 Entry.setLinkage(Linkage);
3990 }
3991 return;
3992 }
3993 OffloadEntriesDeviceGlobalVar.try_emplace(
3994 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3995 ++OffloadingEntriesNum;
3996 }
3997}
3998
3999void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4000 actOnDeviceGlobalVarEntriesInfo(
4001 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4002 // Scan all target region entries and perform the provided action.
4003 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4004 Action(E.getKey(), E.getValue());
4005}
4006
4007llvm::Function *
4008CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
4009 // If we don't have entries or if we are emitting code for the device, we
4010 // don't need to do anything.
4011 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
4012 return nullptr;
4013
4014 llvm::Module &M = CGM.getModule();
4015 ASTContext &C = CGM.getContext();
4016
4017 // Get list of devices we care about
4018 const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
4019
4020 // We should be creating an offloading descriptor only if there are devices
4021 // specified.
4022 assert(!Devices.empty() && "No OpenMP offloading devices??")((!Devices.empty() && "No OpenMP offloading devices??"
) ? static_cast<void> (0) : __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4022, __PRETTY_FUNCTION__))
;
4023
4024 // Create the external variables that will point to the begin and end of the
4025 // host entries section. These will be defined by the linker.
4026 llvm::Type *OffloadEntryTy =
4027 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
4028 auto *HostEntriesBegin = new llvm::GlobalVariable(
4029 M, OffloadEntryTy, /*isConstant=*/true,
4030 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
4031 "__start_omp_offloading_entries");
4032 HostEntriesBegin->setVisibility(llvm::GlobalValue::HiddenVisibility);
4033 auto *HostEntriesEnd = new llvm::GlobalVariable(
4034 M, OffloadEntryTy, /*isConstant=*/true,
4035 llvm::GlobalValue::ExternalLinkage,
4036 /*Initializer=*/nullptr, "__stop_omp_offloading_entries");
4037 HostEntriesEnd->setVisibility(llvm::GlobalValue::HiddenVisibility);
4038
4039 // Create all device images
4040 auto *DeviceImageTy = cast<llvm::StructType>(
4041 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
4042 ConstantInitBuilder DeviceImagesBuilder(CGM);
4043 ConstantArrayBuilder DeviceImagesEntries =
4044 DeviceImagesBuilder.beginArray(DeviceImageTy);
4045
4046 for (const llvm::Triple &Device : Devices) {
4047 StringRef T = Device.getTriple();
4048 std::string BeginName = getName({"omp_offloading", "img_start", ""});
4049 auto *ImgBegin = new llvm::GlobalVariable(
4050 M, CGM.Int8Ty, /*isConstant=*/true,
4051 llvm::GlobalValue::ExternalWeakLinkage,
4052 /*Initializer=*/nullptr, Twine(BeginName).concat(T));
4053 std::string EndName = getName({"omp_offloading", "img_end", ""});
4054 auto *ImgEnd = new llvm::GlobalVariable(
4055 M, CGM.Int8Ty, /*isConstant=*/true,
4056 llvm::GlobalValue::ExternalWeakLinkage,
4057 /*Initializer=*/nullptr, Twine(EndName).concat(T));
4058
4059 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
4060 HostEntriesEnd};
4061 createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
4062 DeviceImagesEntries);
4063 }
4064
4065 // Create device images global array.
4066 std::string ImagesName = getName({"omp_offloading", "device_images"});
4067 llvm::GlobalVariable *DeviceImages =
4068 DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
4069 CGM.getPointerAlign(),
4070 /*isConstant=*/true);
4071 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4072
4073 // This is a Zero array to be used in the creation of the constant expressions
4074 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
4075 llvm::Constant::getNullValue(CGM.Int32Ty)};
4076
4077 // Create the target region descriptor.
4078 llvm::Constant *Data[] = {
4079 llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
4080 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
4081 DeviceImages, Index),
4082 HostEntriesBegin, HostEntriesEnd};
4083 std::string Descriptor = getName({"omp_offloading", "descriptor"});
4084 llvm::GlobalVariable *Desc = createGlobalStruct(
4085 CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
4086
4087 // Emit code to register or unregister the descriptor at execution
4088 // startup or closing, respectively.
4089
4090 llvm::Function *UnRegFn;
4091 {
4092 FunctionArgList Args;
4093 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
4094 Args.push_back(&DummyPtr);
4095
4096 CodeGenFunction CGF(CGM);
4097 // Disable debug info for global (de-)initializer because they are not part
4098 // of some particular construct.
4099 CGF.disableDebugInfo();
4100 const auto &FI =
4101 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4102 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4103 std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
4104 UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
4105 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
4106 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
4107 Desc);
4108 CGF.FinishFunction();
4109 }
4110 llvm::Function *RegFn;
4111 {
4112 CodeGenFunction CGF(CGM);
4113 // Disable debug info for global (de-)initializer because they are not part
4114 // of some particular construct.
4115 CGF.disableDebugInfo();
4116 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
4117 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
4118
4119 // Encode offload target triples into the registration function name. It
4120 // will serve as a comdat key for the registration/unregistration code for
4121 // this particular combination of offloading targets.
4122 SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
4123 RegFnNameParts[0] = "omp_offloading";
4124 RegFnNameParts[1] = "descriptor_reg";
4125 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
4126 [](const llvm::Triple &T) -> const std::string& {
4127 return T.getTriple();
4128 });
4129 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
4130 std::string Descriptor = getName(RegFnNameParts);
4131 RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
4132 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
4133 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
4134 // Create a variable to drive the registration and unregistration of the
4135 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
4136 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
4137 SourceLocation(), nullptr, C.CharTy,
4138 ImplicitParamDecl::Other);
4139 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
4140 CGF.FinishFunction();
4141 }
4142 if (CGM.supportsCOMDAT()) {
4143 // It is sufficient to call registration function only once, so create a
4144 // COMDAT group for registration/unregistration functions and associated
4145 // data. That would reduce startup time and code size. Registration
4146 // function serves as a COMDAT group key.
4147 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
4148 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
4149 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
4150 RegFn->setComdat(ComdatKey);
4151 UnRegFn->setComdat(ComdatKey);
4152 DeviceImages->setComdat(ComdatKey);
4153 Desc->setComdat(ComdatKey);
4154 }
4155 return RegFn;
4156}
4157
4158void CGOpenMPRuntime::createOffloadEntry(
4159 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4160 llvm::GlobalValue::LinkageTypes Linkage) {
4161 StringRef Name = Addr->getName();
4162 llvm::Module &M = CGM.getModule();
4163 llvm::LLVMContext &C = M.getContext();
4164
4165 // Create constant string with the name.
4166 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4167
4168 std::string StringName = getName({"omp_offloading", "entry_name"});
4169 auto *Str = new llvm::GlobalVariable(
4170 M, StrPtrInit->getType(), /*isConstant=*/true,
4171 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4172 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4173
4174 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4175 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4176 llvm::ConstantInt::get(CGM.SizeTy, Size),
4177 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4178 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4179 std::string EntryName = getName({"omp_offloading", "entry", ""});
4180 llvm::GlobalVariable *Entry = createGlobalStruct(
4181 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4182 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4183
4184 // The entry has to be created in the section the linker expects it to be.
4185 Entry->setSection("omp_offloading_entries");
4186}
4187
4188void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4189 // Emit the offloading entries and metadata so that the device codegen side
4190 // can easily figure out what to emit. The produced metadata looks like
4191 // this:
4192 //
4193 // !omp_offload.info = !{!1, ...}
4194 //
4195 // Right now we only generate metadata for function that contain target
4196 // regions.
4197
4198 // If we do not have entries, we don't need to do anything.
4199 if (OffloadEntriesInfoManager.empty())
4200 return;
4201
4202 llvm::Module &M = CGM.getModule();
4203 llvm::LLVMContext &C = M.getContext();
4204 SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
4205 OrderedEntries(OffloadEntriesInfoManager.size());
4206 llvm::SmallVector<StringRef, 16> ParentFunctions(
4207 OffloadEntriesInfoManager.size());
4208
4209 // Auxiliary methods to create metadata values and strings.
4210 auto &&GetMDInt = [this](unsigned V) {
4211 return llvm::ConstantAsMetadata::get(
4212 llvm::ConstantInt::get(CGM.Int32Ty, V));
4213 };
4214
4215 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4216
4217 // Create the offloading info metadata node.
4218 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4219
4220 // Create function that emits metadata for each target region entry;
4221 auto &&TargetRegionMetadataEmitter =
4222 [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
4223 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4224 unsigned Line,
4225 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4226 // Generate metadata for target regions. Each entry of this metadata
4227 // contains:
4228 // - Entry 0 -> Kind of this type of metadata (0).
4229 // - Entry 1 -> Device ID of the file where the entry was identified.
4230 // - Entry 2 -> File ID of the file where the entry was identified.
4231 // - Entry 3 -> Mangled name of the function where the entry was
4232 // identified.
4233 // - Entry 4 -> Line in the file where the entry was identified.
4234 // - Entry 5 -> Order the entry was created.
4235 // The first element of the metadata node is the kind.
4236 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4237 GetMDInt(FileID), GetMDString(ParentName),
4238 GetMDInt(Line), GetMDInt(E.getOrder())};
4239
4240 // Save this entry in the right position of the ordered entries array.
4241 OrderedEntries[E.getOrder()] = &E;
4242 ParentFunctions[E.getOrder()] = ParentName;
4243
4244 // Add metadata to the named metadata node.
4245 MD->addOperand(llvm::MDNode::get(C, Ops));
4246 };
4247
4248 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4249 TargetRegionMetadataEmitter);
4250
4251 // Create function that emits metadata for each device global variable entry;
4252 auto &&DeviceGlobalVarMetadataEmitter =
4253 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4254 MD](StringRef MangledName,
4255 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4256 &E) {
4257 // Generate metadata for global variables. Each entry of this metadata
4258 // contains:
4259 // - Entry 0 -> Kind of this type of metadata (1).
4260 // - Entry 1 -> Mangled name of the variable.
4261 // - Entry 2 -> Declare target kind.
4262 // - Entry 3 -> Order the entry was created.
4263 // The first element of the metadata node is the kind.
4264 llvm::Metadata *Ops[] = {
4265 GetMDInt(E.getKind()), GetMDString(MangledName),
4266 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4267
4268 // Save this entry in the right position of the ordered entries array.
4269 OrderedEntries[E.getOrder()] = &E;
4270
4271 // Add metadata to the named metadata node.
4272 MD->addOperand(llvm::MDNode::get(C, Ops));
4273 };
4274
4275 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4276 DeviceGlobalVarMetadataEmitter);
4277
4278 for (const auto *E : OrderedEntries) {
4279 assert(E && "All ordered entries must exist!")((E && "All ordered entries must exist!") ? static_cast
<void> (0) : __assert_fail ("E && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4279, __PRETTY_FUNCTION__))
;
4280 if (const auto *CE =
4281 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4282 E)) {
4283 if (!CE->getID() || !CE->getAddress()) {
4284 // Do not blame the entry if the parent funtion is not emitted.
4285 StringRef FnName = ParentFunctions[CE->getOrder()];
4286 if (!CGM.GetGlobalValue(FnName))
4287 continue;
4288 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4289 DiagnosticsEngine::Error,
4290 "Offloading entry for target region is incorrect: either the "
4291 "address or the ID is invalid.");
4292 CGM.getDiags().Report(DiagID);
4293 continue;
4294 }
4295 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4296 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4297 } else if (const auto *CE =
4298 dyn_cast<OffloadEntriesInfoManagerTy::
4299 OffloadEntryInfoDeviceGlobalVar>(E)) {
4300 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4301 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4302 CE->getFlags());
4303 switch (Flags) {
4304 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4305 if (CGM.getLangOpts().OpenMPIsDevice &&
4306 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4307 continue;
4308 if (!CE->getAddress()) {
4309 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4310 DiagnosticsEngine::Error,
4311 "Offloading entry for declare target variable is incorrect: the "
4312 "address is invalid.");
4313 CGM.getDiags().Report(DiagID);
4314 continue;
4315 }
4316 // The vaiable has no definition - no need to add the entry.
4317 if (CE->getVarSize().isZero())
4318 continue;
4319 break;
4320 }
4321 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4322 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
4323 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
4324 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4324, __PRETTY_FUNCTION__))
;
4325 if (CGM.getLangOpts().OpenMPIsDevice)
4326 continue;
4327 if (!CE->getAddress()) {
4328 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4329 DiagnosticsEngine::Error,
4330 "Offloading entry for declare target variable is incorrect: the "
4331 "address is invalid.");
4332 CGM.getDiags().Report(DiagID);
4333 continue;
4334 }
4335 break;
4336 }
4337 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4338 CE->getVarSize().getQuantity(), Flags,
4339 CE->getLinkage());
4340 } else {
4341 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4341)
;
4342 }
4343 }
4344}
4345
4346/// Loads all the offload entries information from the host IR
4347/// metadata.
4348void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4349 // If we are in target mode, load the metadata from the host IR. This code has
4350 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4351
4352 if (!CGM.getLangOpts().OpenMPIsDevice)
4353 return;
4354
4355 if (CGM.getLangOpts().OMPHostIRFile.empty())
4356 return;
4357
4358 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4359 if (auto EC = Buf.getError()) {
4360 CGM.getDiags().Report(diag::err_cannot_open_file)
4361 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4362 return;
4363 }
4364
4365 llvm::LLVMContext C;
4366 auto ME = expectedToErrorOrAndEmitErrors(
4367 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4368
4369 if (auto EC = ME.getError()) {
4370 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4371 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4372 CGM.getDiags().Report(DiagID)
4373 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4374 return;
4375 }
4376
4377 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4378 if (!MD)
4379 return;
4380
4381 for (llvm::MDNode *MN : MD->operands()) {
4382 auto &&GetMDInt = [MN](unsigned Idx) {
4383 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4384 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4385 };
4386
4387 auto &&GetMDString = [MN](unsigned Idx) {
4388 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4389 return V->getString();
4390 };
4391
4392 switch (GetMDInt(0)) {
4393 default:
4394 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4394)
;
4395 break;
4396 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4397 OffloadingEntryInfoTargetRegion:
4398 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4399 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4400 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4401 /*Order=*/GetMDInt(5));
4402 break;
4403 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4404 OffloadingEntryInfoDeviceGlobalVar:
4405 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4406 /*MangledName=*/GetMDString(1),
4407 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4408 /*Flags=*/GetMDInt(2)),
4409 /*Order=*/GetMDInt(3));
4410 break;
4411 }
4412 }
4413}
4414
4415void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4416 if (!KmpRoutineEntryPtrTy) {
4417 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4418 ASTContext &C = CGM.getContext();
4419 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4420 FunctionProtoType::ExtProtoInfo EPI;
4421 KmpRoutineEntryPtrQTy = C.getPointerType(
4422 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4423 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4424 }
4425}
4426
4427QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4428 // Make sure the type of the entry is already created. This is the type we
4429 // have to create:
4430 // struct __tgt_offload_entry{
4431 // void *addr; // Pointer to the offload entry info.
4432 // // (function or global)
4433 // char *name; // Name of the function or global.
4434 // size_t size; // Size of the entry info (0 if it a function).
4435 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4436 // int32_t reserved; // Reserved, to use by the runtime library.
4437 // };
4438 if (TgtOffloadEntryQTy.isNull()) {
4439 ASTContext &C = CGM.getContext();
4440 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4441 RD->startDefinition();
4442 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4443 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4444 addFieldToRecordDecl(C, RD, C.getSizeType());
4445 addFieldToRecordDecl(
4446 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4447 addFieldToRecordDecl(
4448 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4449 RD->completeDefinition();
4450 RD->addAttr(PackedAttr::CreateImplicit(C));
4451 TgtOffloadEntryQTy = C.getRecordType(RD);
4452 }
4453 return TgtOffloadEntryQTy;
4454}
4455
4456QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4457 // These are the types we need to build:
4458 // struct __tgt_device_image{
4459 // void *ImageStart; // Pointer to the target code start.
4460 // void *ImageEnd; // Pointer to the target code end.
4461 // // We also add the host entries to the device image, as it may be useful
4462 // // for the target runtime to have access to that information.
4463 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
4464 // // the entries.
4465 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4466 // // entries (non inclusive).
4467 // };
4468 if (TgtDeviceImageQTy.isNull()) {
4469 ASTContext &C = CGM.getContext();
4470 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4471 RD->startDefinition();
4472 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4473 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4474 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4475 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4476 RD->completeDefinition();
4477 TgtDeviceImageQTy = C.getRecordType(RD);
4478 }
4479 return TgtDeviceImageQTy;
4480}
4481
4482QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4483 // struct __tgt_bin_desc{
4484 // int32_t NumDevices; // Number of devices supported.
4485 // __tgt_device_image *DeviceImages; // Arrays of device images
4486 // // (one per device).
4487 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
4488 // // entries.
4489 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4490 // // entries (non inclusive).
4491 // };
4492 if (TgtBinaryDescriptorQTy.isNull()) {
4493 ASTContext &C = CGM.getContext();
4494 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4495 RD->startDefinition();
4496 addFieldToRecordDecl(
4497 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4498 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4499 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4500 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4501 RD->completeDefinition();
4502 TgtBinaryDescriptorQTy = C.getRecordType(RD);
4503 }
4504 return TgtBinaryDescriptorQTy;
4505}
4506
4507namespace {
4508struct PrivateHelpersTy {
4509 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4510 const VarDecl *PrivateElemInit)
4511 : Original(Original), PrivateCopy(PrivateCopy),
4512 PrivateElemInit(PrivateElemInit) {}
4513 const VarDecl *Original;
4514 const VarDecl *PrivateCopy;
4515 const VarDecl *PrivateElemInit;
4516};
4517typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4518} // anonymous namespace
4519
4520static RecordDecl *
4521createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4522 if (!Privates.empty()) {
4523 ASTContext &C = CGM.getContext();
4524 // Build struct .kmp_privates_t. {
4525 // /* private vars */
4526 // };
4527 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4528 RD->startDefinition();
4529 for (const auto &Pair : Privates) {
4530 const VarDecl *VD = Pair.second.Original;
4531 QualType Type = VD->getType().getNonReferenceType();
4532 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4533 if (VD->hasAttrs()) {
4534 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4535 E(VD->getAttrs().end());
4536 I != E; ++I)
4537 FD->addAttr(*I);
4538 }
4539 }
4540 RD->completeDefinition();
4541 return RD;
4542 }
4543 return nullptr;
4544}
4545
4546static RecordDecl *
4547createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4548 QualType KmpInt32Ty,
4549 QualType KmpRoutineEntryPointerQTy) {
4550 ASTContext &C = CGM.getContext();
4551 // Build struct kmp_task_t {
4552 // void * shareds;
4553 // kmp_routine_entry_t routine;
4554 // kmp_int32 part_id;
4555 // kmp_cmplrdata_t data1;
4556 // kmp_cmplrdata_t data2;
4557 // For taskloops additional fields:
4558 // kmp_uint64 lb;
4559 // kmp_uint64 ub;
4560 // kmp_int64 st;
4561 // kmp_int32 liter;
4562 // void * reductions;
4563 // };
4564 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4565 UD->startDefinition();
4566 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4567 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4568 UD->completeDefinition();
4569 QualType KmpCmplrdataTy = C.getRecordType(UD);
4570 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4571 RD->startDefinition();
4572 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4573 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4574 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4575 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4576 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4577 if (isOpenMPTaskLoopDirective(Kind)) {
4578 QualType KmpUInt64Ty =
4579 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4580 QualType KmpInt64Ty =
4581 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4582 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4583 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4584 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4585 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4586 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4587 }
4588 RD->completeDefinition();
4589 return RD;
4590}
4591
4592static RecordDecl *
4593createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4594 ArrayRef<PrivateDataTy> Privates) {
4595 ASTContext &C = CGM.getContext();
4596 // Build struct kmp_task_t_with_privates {
4597 // kmp_task_t task_data;
4598 // .kmp_privates_t. privates;
4599 // };
4600 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4601 RD->startDefinition();
4602 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4603 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4604 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4605 RD->completeDefinition();
4606 return RD;
4607}
4608
4609/// Emit a proxy function which accepts kmp_task_t as the second
4610/// argument.
4611/// \code
4612/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4613/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4614/// For taskloops:
4615/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4616/// tt->reductions, tt->shareds);
4617/// return 0;
4618/// }
4619/// \endcode
4620static llvm::Function *
4621emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4622 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4623 QualType KmpTaskTWithPrivatesPtrQTy,
4624 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4625 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4626 llvm::Value *TaskPrivatesMap) {
4627 ASTContext &C = CGM.getContext();
4628 FunctionArgList Args;
4629 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4630 ImplicitParamDecl::Other);
4631 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4632 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4633 ImplicitParamDecl::Other);
4634 Args.push_back(&GtidArg);
4635 Args.push_back(&TaskTypeArg);
4636 const auto &TaskEntryFnInfo =
4637 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4638 llvm::FunctionType *TaskEntryTy =
4639 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4640 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4641 auto *TaskEntry = llvm::Function::Create(
4642 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4643 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4644 TaskEntry->setDoesNotRecurse();
4645 CodeGenFunction CGF(CGM);
4646 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4647 Loc, Loc);
4648
4649 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4650 // tt,
4651 // For taskloops:
4652 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4653 // tt->task_data.shareds);
4654 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4655 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4656 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4657 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4658 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4659 const auto *KmpTaskTWithPrivatesQTyRD =
4660 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4661 LValue Base =
4662 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4663 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4664 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4665 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4666 llvm::Value *PartidParam = PartIdLVal.getPointer();
4667
4668 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4669 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4670 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4671 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4672 CGF.ConvertTypeForMem(SharedsPtrTy));
4673
4674 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4675 llvm::Value *PrivatesParam;
4676 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4677 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4678 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4679 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4680 } else {
4681 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4682 }
4683
4684 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4685 TaskPrivatesMap,
4686 CGF.Builder
4687 .CreatePointerBitCastOrAddrSpaceCast(
4688 TDBase.getAddress(), CGF.VoidPtrTy)
4689 .getPointer()};
4690 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4691 std::end(CommonArgs));
4692 if (isOpenMPTaskLoopDirective(Kind)) {
4693 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4694 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4695 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4696 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4697 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4698 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4699 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4700 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4701 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4702 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4703 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4704 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4705 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4706 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4707 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4708 CallArgs.push_back(LBParam);
4709 CallArgs.push_back(UBParam);
4710 CallArgs.push_back(StParam);
4711 CallArgs.push_back(LIParam);
4712 CallArgs.push_back(RParam);
4713 }
4714 CallArgs.push_back(SharedsParam);
4715
4716 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4717 CallArgs);
4718 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4719 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4720 CGF.FinishFunction();
4721 return TaskEntry;
4722}
4723
4724static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4725 SourceLocation Loc,
4726 QualType KmpInt32Ty,
4727 QualType KmpTaskTWithPrivatesPtrQTy,
4728 QualType KmpTaskTWithPrivatesQTy) {
4729 ASTContext &C = CGM.getContext();
4730 FunctionArgList Args;
4731 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4732 ImplicitParamDecl::Other);
4733 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4734 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4735 ImplicitParamDecl::Other);
4736 Args.push_back(&GtidArg);
4737 Args.push_back(&TaskTypeArg);
4738 const auto &DestructorFnInfo =
4739 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4740 llvm::FunctionType *DestructorFnTy =
4741 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4742 std::string Name =
4743 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4744 auto *DestructorFn =
4745 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4746 Name, &CGM.getModule());
4747 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4748 DestructorFnInfo);
4749 DestructorFn->setDoesNotRecurse();
4750 CodeGenFunction CGF(CGM);
4751 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4752 Args, Loc, Loc);
4753
4754 LValue Base = CGF.EmitLoadOfPointerLValue(
4755 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4756 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4757 const auto *KmpTaskTWithPrivatesQTyRD =
4758 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4759 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4760 Base = CGF.EmitLValueForField(Base, *FI);
4761 for (const auto *Field :
4762 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4763 if (QualType::DestructionKind DtorKind =
4764 Field->getType().isDestructedType()) {
4765 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4766 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4767 }
4768 }
4769 CGF.FinishFunction();
4770 return DestructorFn;
4771}
4772
4773/// Emit a privates mapping function for correct handling of private and
4774/// firstprivate variables.
4775/// \code
4776/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4777/// **noalias priv1,..., <tyn> **noalias privn) {
4778/// *priv1 = &.privates.priv1;
4779/// ...;
4780/// *privn = &.privates.privn;
4781/// }
4782/// \endcode
4783static llvm::Value *
4784emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4785 ArrayRef<const Expr *> PrivateVars,
4786 ArrayRef<const Expr *> FirstprivateVars,
4787 ArrayRef<const Expr *> LastprivateVars,
4788 QualType PrivatesQTy,
4789 ArrayRef<PrivateDataTy> Privates) {
4790 ASTContext &C = CGM.getContext();
4791 FunctionArgList Args;
4792 ImplicitParamDecl TaskPrivatesArg(
4793 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4794 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4795 ImplicitParamDecl::Other);
4796 Args.push_back(&TaskPrivatesArg);
4797 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4798 unsigned Counter = 1;
4799 for (const Expr *E : PrivateVars) {
4800 Args.push_back(ImplicitParamDecl::Create(
4801 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4802 C.getPointerType(C.getPointerType(E->getType()))
4803 .withConst()
4804 .withRestrict(),
4805 ImplicitParamDecl::Other));
4806 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4807 PrivateVarsPos[VD] = Counter;
4808 ++Counter;
4809 }
4810 for (const Expr *E : FirstprivateVars) {
4811 Args.push_back(ImplicitParamDecl::Create(
4812 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4813 C.getPointerType(C.getPointerType(E->getType()))
4814 .withConst()
4815 .withRestrict(),
4816 ImplicitParamDecl::Other));
4817 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4818 PrivateVarsPos[VD] = Counter;
4819 ++Counter;
4820 }
4821 for (const Expr *E : LastprivateVars) {
4822 Args.push_back(ImplicitParamDecl::Create(
4823 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4824 C.getPointerType(C.getPointerType(E->getType()))
4825 .withConst()
4826 .withRestrict(),
4827 ImplicitParamDecl::Other));
4828 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4829 PrivateVarsPos[VD] = Counter;
4830 ++Counter;
4831 }
4832 const auto &TaskPrivatesMapFnInfo =
4833 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4834 llvm::FunctionType *TaskPrivatesMapTy =
4835 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4836 std::string Name =
4837 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4838 auto *TaskPrivatesMap = llvm::Function::Create(
4839 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4840 &CGM.getModule());
4841 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4842 TaskPrivatesMapFnInfo);
4843 if (CGM.getLangOpts().Optimize) {
4844 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4845 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4846 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4847 }
4848 CodeGenFunction CGF(CGM);
4849 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4850 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4851
4852 // *privi = &.privates.privi;
4853 LValue Base = CGF.EmitLoadOfPointerLValue(
4854 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4855 TaskPrivatesArg.getType()->castAs<PointerType>());
4856 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4857 Counter = 0;
4858 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4859 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4860 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4861 LValue RefLVal =
4862 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4863 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4864 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4865 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4866 ++Counter;
4867 }
4868 CGF.FinishFunction();
4869 return TaskPrivatesMap;
4870}
4871
4872/// Emit initialization for private variables in task-based directives.
4873static void emitPrivatesInit(CodeGenFunction &CGF,
4874 const OMPExecutableDirective &D,
4875 Address KmpTaskSharedsPtr, LValue TDBase,
4876 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4877 QualType SharedsTy, QualType SharedsPtrTy,
4878 const OMPTaskDataTy &Data,
4879 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4880 ASTContext &C = CGF.getContext();
4881 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4882 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4883 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4884 ? OMPD_taskloop
4885 : OMPD_task;
4886 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4887 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4888 LValue SrcBase;
4889 bool IsTargetTask =
4890 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4891 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4892 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4893 // PointersArray and SizesArray. The original variables for these arrays are
4894 // not captured and we get their addresses explicitly.
4895 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4896 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4897 SrcBase = CGF.MakeAddrLValue(
4898 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4899 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4900 SharedsTy);
4901 }
4902 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4903 for (const PrivateDataTy &Pair : Privates) {
4904 const VarDecl *VD = Pair.second.PrivateCopy;
4905 const Expr *Init = VD->getAnyInitializer();
4906 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4907 !CGF.isTrivialInitializer(Init)))) {
4908 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4909 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4910 const VarDecl *OriginalVD = Pair.second.Original;
4911 // Check if the variable is the target-based BasePointersArray,
4912 // PointersArray or SizesArray.
4913 LValue SharedRefLValue;
4914 QualType Type = PrivateLValue.getType();
4915 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4916 if (IsTargetTask && !SharedField) {
4917 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4918 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4919 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4920 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4921 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4922 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4923 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
4924 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4924, __PRETTY_FUNCTION__))
;
4925 SharedRefLValue =
4926 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4927 } else {
4928 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4929 SharedRefLValue = CGF.MakeAddrLValue(
4930 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4931 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4932 SharedRefLValue.getTBAAInfo());
4933 }
4934 if (Type->isArrayType()) {
4935 // Initialize firstprivate array.
4936 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4937 // Perform simple memcpy.
4938 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4939 } else {
4940 // Initialize firstprivate array using element-by-element
4941 // initialization.
4942 CGF.EmitOMPAggregateAssign(
4943 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4944 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4945 Address SrcElement) {
4946 // Clean up any temporaries needed by the initialization.
4947 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4948 InitScope.addPrivate(
4949 Elem, [SrcElement]() -> Address { return SrcElement; });
4950 (void)InitScope.Privatize();
4951 // Emit initialization for single element.
4952 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4953 CGF, &CapturesInfo);
4954 CGF.EmitAnyExprToMem(Init, DestElement,
4955 Init->getType().getQualifiers(),
4956 /*IsInitializer=*/false);
4957 });
4958 }
4959 } else {
4960 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4961 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4962 return SharedRefLValue.getAddress();
4963 });
4964 (void)InitScope.Privatize();
4965 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4966 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4967 /*capturedByInit=*/false);
4968 }
4969 } else {
4970 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4971 }
4972 }
4973 ++FI;
4974 }
4975}
4976
4977/// Check if duplication function is required for taskloops.
4978static bool checkInitIsRequired(CodeGenFunction &CGF,
4979 ArrayRef<PrivateDataTy> Privates) {
4980 bool InitRequired = false;
4981 for (const PrivateDataTy &Pair : Privates) {
4982 const VarDecl *VD = Pair.second.PrivateCopy;
4983 const Expr *Init = VD->getAnyInitializer();
4984 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4985 !CGF.isTrivialInitializer(Init));
4986 if (InitRequired)
4987 break;
4988 }
4989 return InitRequired;
4990}
4991
4992
4993/// Emit task_dup function (for initialization of
4994/// private/firstprivate/lastprivate vars and last_iter flag)
4995/// \code
4996/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4997/// lastpriv) {
4998/// // setup lastprivate flag
4999/// task_dst->last = lastpriv;
5000/// // could be constructor calls here...
5001/// }
5002/// \endcode
5003static llvm::Value *
5004emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
5005 const OMPExecutableDirective &D,
5006 QualType KmpTaskTWithPrivatesPtrQTy,
5007 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
5008 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
5009 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
5010 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
5011 ASTContext &C = CGM.getContext();
5012 FunctionArgList Args;
5013 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5014 KmpTaskTWithPrivatesPtrQTy,
5015 ImplicitParamDecl::Other);
5016 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5017 KmpTaskTWithPrivatesPtrQTy,
5018 ImplicitParamDecl::Other);
5019 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
5020 ImplicitParamDecl::Other);
5021 Args.push_back(&DstArg);
5022 Args.push_back(&SrcArg);
5023 Args.push_back(&LastprivArg);
5024 const auto &TaskDupFnInfo =
5025 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5026 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
5027 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
5028 auto *TaskDup = llvm::Function::Create(
5029 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
5030 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
5031 TaskDup->setDoesNotRecurse();
5032 CodeGenFunction CGF(CGM);
5033 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
5034 Loc);
5035
5036 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5037 CGF.GetAddrOfLocalVar(&DstArg),
5038 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5039 // task_dst->liter = lastpriv;
5040 if (WithLastIter) {
5041 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
5042 LValue Base = CGF.EmitLValueForField(
5043 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5044 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
5045 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
5046 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
5047 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
5048 }
5049
5050 // Emit initial values for private copies (if any).
5051 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5051, __PRETTY_FUNCTION__))
;
5052 Address KmpTaskSharedsPtr = Address::invalid();
5053 if (!Data.FirstprivateVars.empty()) {
5054 LValue TDBase = CGF.EmitLoadOfPointerLValue(
5055 CGF.GetAddrOfLocalVar(&SrcArg),
5056 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
5057 LValue Base = CGF.EmitLValueForField(
5058 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
5059 KmpTaskSharedsPtr = Address(
5060 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
5061 Base, *std::next(KmpTaskTQTyRD->field_begin(),
5062 KmpTaskTShareds)),
5063 Loc),
5064 CGF.getNaturalTypeAlignment(SharedsTy));
5065 }
5066 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
5067 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
5068 CGF.FinishFunction();
5069 return TaskDup;
5070}
5071
5072/// Checks if destructor function is required to be generated.
5073/// \return true if cleanups are required, false otherwise.
5074static bool
5075checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
5076 bool NeedsCleanup = false;
5077 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
5078 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
5079 for (const FieldDecl *FD : PrivateRD->fields()) {
5080 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
5081 if (NeedsCleanup)
5082 break;
5083 }
5084 return NeedsCleanup;
5085}
5086
5087CGOpenMPRuntime::TaskResultTy
5088CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
5089 const OMPExecutableDirective &D,
5090 llvm::Function *TaskFunction, QualType SharedsTy,
5091 Address Shareds, const OMPTaskDataTy &Data) {
5092 ASTContext &C = CGM.getContext();
5093 llvm::SmallVector<PrivateDataTy, 4> Privates;
5094 // Aggregate privates and sort them by the alignment.
5095 auto I = Data.PrivateCopies.begin();
5096 for (const Expr *E : Data.PrivateVars) {
5097 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5098 Privates.emplace_back(
5099 C.getDeclAlign(VD),
5100 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5101 /*PrivateElemInit=*/nullptr));
5102 ++I;
5103 }
5104 I = Data.FirstprivateCopies.begin();
5105 auto IElemInitRef = Data.FirstprivateInits.begin();
5106 for (const Expr *E : Data.FirstprivateVars) {
5107 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5108 Privates.emplace_back(
5109 C.getDeclAlign(VD),
5110 PrivateHelpersTy(
5111 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5112 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
5113 ++I;
5114 ++IElemInitRef;
5115 }
5116 I = Data.LastprivateCopies.begin();
5117 for (const Expr *E : Data.LastprivateVars) {
5118 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5119 Privates.emplace_back(
5120 C.getDeclAlign(VD),
5121 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5122 /*PrivateElemInit=*/nullptr));
5123 ++I;
5124 }
5125 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5126 return L.first > R.first;
5127 });
5128 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5129 // Build type kmp_routine_entry_t (if not built yet).
5130 emitKmpRoutineEntryT(KmpInt32Ty);
5131 // Build type kmp_task_t (if not built yet).
5132 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5133 if (SavedKmpTaskloopTQTy.isNull()) {
5134 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5135 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5136 }
5137 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5138 } else {
5139 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5140 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5141 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
5142 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5142, __PRETTY_FUNCTION__))
;
5143 if (SavedKmpTaskTQTy.isNull()) {
5144 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5145 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5146 }
5147 KmpTaskTQTy = SavedKmpTaskTQTy;
5148 }
5149 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5150 // Build particular struct kmp_task_t for the given task.
5151 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5152 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5153 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5154 QualType KmpTaskTWithPrivatesPtrQTy =
5155 C.getPointerType(KmpTaskTWithPrivatesQTy);
5156 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5157 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5158 KmpTaskTWithPrivatesTy->getPointerTo();
5159 llvm::Value *KmpTaskTWithPrivatesTySize =
5160 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5161 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5162
5163 // Emit initial values for private copies (if any).
5164 llvm::Value *TaskPrivatesMap = nullptr;
5165 llvm::Type *TaskPrivatesMapTy =
5166 std::next(TaskFunction->arg_begin(), 3)->getType();
5167 if (!Privates.empty()) {
5168 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5169 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5170 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5171 FI->getType(), Privates);
5172 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5173 TaskPrivatesMap, TaskPrivatesMapTy);
5174 } else {
5175 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5176 cast<llvm::PointerType>(TaskPrivatesMapTy));
5177 }
5178 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5179 // kmp_task_t *tt);
5180 llvm::Function *TaskEntry = emitProxyTaskFunction(
5181 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5182 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5183 TaskPrivatesMap);
5184
5185 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5186 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5187 // kmp_routine_entry_t *task_entry);
5188 // Task flags. Format is taken from
5189 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5190 // description of kmp_tasking_flags struct.
5191 enum {
5192 TiedFlag = 0x1,
5193 FinalFlag = 0x2,
5194 DestructorsFlag = 0x8,
5195 PriorityFlag = 0x20
5196 };
5197 unsigned Flags = Data.Tied ? TiedFlag : 0;
5198 bool NeedsCleanup = false;
5199 if (!Privates.empty()) {
5200 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5201 if (NeedsCleanup)
5202 Flags = Flags | DestructorsFlag;
5203 }
5204 if (Data.Priority.getInt())
5205 Flags = Flags | PriorityFlag;
5206 llvm::Value *TaskFlags =
5207 Data.Final.getPointer()
5208 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5209 CGF.Builder.getInt32(FinalFlag),
5210 CGF.Builder.getInt32(/*C=*/0))
5211 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5212 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5213 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5214 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5215 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5216 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5217 TaskEntry, KmpRoutineEntryPtrTy)};
5218 llvm::Value *NewTask;
5219 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5220 // Check if we have any device clause associated with the directive.
5221 const Expr *Device = nullptr;
5222 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5223 Device = C->getDevice();
5224 // Emit device ID if any otherwise use default value.
5225 llvm::Value *DeviceID;
5226 if (Device)
5227 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5228 CGF.Int64Ty, /*isSigned=*/true);
5229 else
5230 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5231 AllocArgs.push_back(DeviceID);
5232 NewTask = CGF.EmitRuntimeCall(
5233 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5234 } else {
5235 NewTask = CGF.EmitRuntimeCall(
5236 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5237 }
5238 llvm::Value *NewTaskNewTaskTTy =
5239 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5240 NewTask, KmpTaskTWithPrivatesPtrTy);
5241 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5242 KmpTaskTWithPrivatesQTy);
5243 LValue TDBase =
5244 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5245 // Fill the data in the resulting kmp_task_t record.
5246 // Copy shareds if there are any.
5247 Address KmpTaskSharedsPtr = Address::invalid();
5248 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5249 KmpTaskSharedsPtr =
5250 Address(CGF.EmitLoadOfScalar(
5251 CGF.EmitLValueForField(
5252 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5253 KmpTaskTShareds)),
5254 Loc),
5255 CGF.getNaturalTypeAlignment(SharedsTy));
5256 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5257 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5258 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5259 }
5260 // Emit initial values for private copies (if any).
5261 TaskResultTy Result;
5262 if (!Privates.empty()) {
5263 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5264 SharedsTy, SharedsPtrTy, Data, Privates,
5265 /*ForDup=*/false);
5266 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5267 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5268 Result.TaskDupFn = emitTaskDupFunction(
5269 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5270 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5271 /*WithLastIter=*/!Data.LastprivateVars.empty());
5272 }
5273 }
5274 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5275 enum { Priority = 0, Destructors = 1 };
5276 // Provide pointer to function with destructors for privates.
5277 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5278 const RecordDecl *KmpCmplrdataUD =
5279 (*FI)->getType()->getAsUnionType()->getDecl();
5280 if (NeedsCleanup) {
5281 llvm::Value *DestructorFn = emitDestructorsFunction(
5282 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5283 KmpTaskTWithPrivatesQTy);
5284 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5285 LValue DestructorsLV = CGF.EmitLValueForField(
5286 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5287 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5288 DestructorFn, KmpRoutineEntryPtrTy),
5289 DestructorsLV);
5290 }
5291 // Set priority.
5292 if (Data.Priority.getInt()) {
5293 LValue Data2LV = CGF.EmitLValueForField(
5294 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5295 LValue PriorityLV = CGF.EmitLValueForField(
5296 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5297 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5298 }
5299 Result.NewTask = NewTask;
5300 Result.TaskEntry = TaskEntry;
5301 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5302 Result.TDBase = TDBase;
5303 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5304 return Result;
5305}
5306
5307void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5308 const OMPExecutableDirective &D,
5309 llvm::Function *TaskFunction,
5310 QualType SharedsTy, Address Shareds,
5311 const Expr *IfCond,
5312 const OMPTaskDataTy &Data) {
5313 if (!CGF.HaveInsertPoint())
5314 return;
5315
5316 TaskResultTy Result =
5317 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5318 llvm::Value *NewTask = Result.NewTask;
5319 llvm::Function *TaskEntry = Result.TaskEntry;
5320 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5321 LValue TDBase = Result.TDBase;
5322 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5323 ASTContext &C = CGM.getContext();
5324 // Process list of dependences.
5325 Address DependenciesArray = Address::invalid();
5326 unsigned NumDependencies = Data.Dependences.size();
5327 if (NumDependencies) {
5328 // Dependence kind for RTL.
5329 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5330 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5331 RecordDecl *KmpDependInfoRD;
5332 QualType FlagsTy =
5333 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5334 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5335 if (KmpDependInfoTy.isNull()) {
5336 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5337 KmpDependInfoRD->startDefinition();
5338 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5339 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5340 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5341 KmpDependInfoRD->completeDefinition();
5342 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5343 } else {
5344 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5345 }
5346 // Define type kmp_depend_info[<Dependences.size()>];
5347 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5348 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5349 ArrayType::Normal, /*IndexTypeQuals=*/0);
5350 // kmp_depend_info[<Dependences.size()>] deps;
5351 DependenciesArray =
5352 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5353 for (unsigned I = 0; I < NumDependencies; ++I) {
5354 const Expr *E = Data.Dependences[I].second;
5355 LValue Addr = CGF.EmitLValue(E);
5356 llvm::Value *Size;
5357 QualType Ty = E->getType();
5358 if (const auto *ASE =
5359 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5360 LValue UpAddrLVal =
5361 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5362 llvm::Value *UpAddr =
5363 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5364 llvm::Value *LowIntPtr =
5365 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5366 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5367 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5368 } else {
5369 Size = CGF.getTypeSize(Ty);
5370 }
5371 LValue Base = CGF.MakeAddrLValue(
5372 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5373 KmpDependInfoTy);
5374 // deps[i].base_addr = &<Dependences[i].second>;
5375 LValue BaseAddrLVal = CGF.EmitLValueForField(
5376 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5377 CGF.EmitStoreOfScalar(
5378 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5379 BaseAddrLVal);
5380 // deps[i].len = sizeof(<Dependences[i].second>);
5381 LValue LenLVal = CGF.EmitLValueForField(
5382 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5383 CGF.EmitStoreOfScalar(Size, LenLVal);
5384 // deps[i].flags = <Dependences[i].first>;
5385 RTLDependenceKindTy DepKind;
5386 switch (Data.Dependences[I].first) {
5387 case OMPC_DEPEND_in:
5388 DepKind = DepIn;
5389 break;
5390 // Out and InOut dependencies must use the same code.
5391 case OMPC_DEPEND_out:
5392 case OMPC_DEPEND_inout:
5393 DepKind = DepInOut;
5394 break;
5395 case OMPC_DEPEND_mutexinoutset:
5396 DepKind = DepMutexInOutSet;
5397 break;
5398 case OMPC_DEPEND_source:
5399 case OMPC_DEPEND_sink:
5400 case OMPC_DEPEND_unknown:
5401 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5401)
;
5402 }
5403 LValue FlagsLVal = CGF.EmitLValueForField(
5404 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5405 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5406 FlagsLVal);
5407 }
5408 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5409 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5410 }
5411
5412 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5413 // libcall.
5414 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5415 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5416 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5417 // list is not empty
5418 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5419 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5420 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5421 llvm::Value *DepTaskArgs[7];
5422 if (NumDependencies) {
5423 DepTaskArgs[0] = UpLoc;
5424 DepTaskArgs[1] = ThreadID;
5425 DepTaskArgs[2] = NewTask;
5426 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5427 DepTaskArgs[4] = DependenciesArray.getPointer();
5428 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5429 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5430 }
5431 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5432 &TaskArgs,
5433 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5434 if (!Data.Tied) {
5435 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5436 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5437 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5438 }
5439 if (NumDependencies) {
5440 CGF.EmitRuntimeCall(
5441 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5442 } else {
5443 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5444 TaskArgs);
5445 }
5446 // Check if parent region is untied and build return for untied task;
5447 if (auto *Region =
5448 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5449 Region->emitUntiedSwitch(CGF);
5450 };
5451
5452 llvm::Value *DepWaitTaskArgs[6];
5453 if (NumDependencies) {
5454 DepWaitTaskArgs[0] = UpLoc;
5455 DepWaitTaskArgs[1] = ThreadID;
5456 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5457 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5458 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5459 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5460 }
5461 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5462 NumDependencies, &DepWaitTaskArgs,
5463 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5464 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5465 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5466 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5467 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5468 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5469 // is specified.
5470 if (NumDependencies)
5471 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5472 DepWaitTaskArgs);
5473 // Call proxy_task_entry(gtid, new_task);
5474 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5475 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5476 Action.Enter(CGF);
5477 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5478 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5479 OutlinedFnArgs);
5480 };
5481
5482 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5483 // kmp_task_t *new_task);
5484 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5485 // kmp_task_t *new_task);
5486 RegionCodeGenTy RCG(CodeGen);
5487 CommonActionTy Action(
5488 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5489 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5490 RCG.setAction(Action);
5491 RCG(CGF);
5492 };
5493
5494 if (IfCond) {
5495 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5496 } else {
5497 RegionCodeGenTy ThenRCG(ThenCodeGen);
5498 ThenRCG(CGF);
5499 }
5500}
5501
5502void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5503 const OMPLoopDirective &D,
5504 llvm::Function *TaskFunction,
5505 QualType SharedsTy, Address Shareds,
5506 const Expr *IfCond,
5507 const OMPTaskDataTy &Data) {
5508 if (!CGF.HaveInsertPoint())
5509 return;
5510 TaskResultTy Result =
5511 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5512 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5513 // libcall.
5514 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5515 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5516 // sched, kmp_uint64 grainsize, void *task_dup);
5517 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5518 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5519 llvm::Value *IfVal;
5520 if (IfCond) {
5521 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5522 /*isSigned=*/true);
5523 } else {
5524 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5525 }
5526
5527 LValue LBLVal = CGF.EmitLValueForField(
5528 Result.TDBase,
5529 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5530 const auto *LBVar =
5531 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5532 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5533 /*IsInitializer=*/true);
5534 LValue UBLVal = CGF.EmitLValueForField(
5535 Result.TDBase,
5536 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5537 const auto *UBVar =
5538 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5539 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5540 /*IsInitializer=*/true);
5541 LValue StLVal = CGF.EmitLValueForField(
5542 Result.TDBase,
5543 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5544 const auto *StVar =
5545 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5546 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5547 /*IsInitializer=*/true);
5548 // Store reductions address.
5549 LValue RedLVal = CGF.EmitLValueForField(
5550 Result.TDBase,
5551 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5552 if (Data.Reductions) {
5553 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5554 } else {
5555 CGF.EmitNullInitialization(RedLVal.getAddress(),
5556 CGF.getContext().VoidPtrTy);
5557 }
5558 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5559 llvm::Value *TaskArgs[] = {
5560 UpLoc,
5561 ThreadID,
5562 Result.NewTask,
5563 IfVal,
5564 LBLVal.getPointer(),
5565 UBLVal.getPointer(),
5566 CGF.EmitLoadOfScalar(StLVal, Loc),
5567 llvm::ConstantInt::getSigned(
5568 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5569 llvm::ConstantInt::getSigned(
5570 CGF.IntTy, Data.Schedule.getPointer()
5571 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5572 : NoSchedule),
5573 Data.Schedule.getPointer()
5574 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5575 /*isSigned=*/false)
5576 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5577 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5578 Result.TaskDupFn, CGF.VoidPtrTy)
5579 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5580 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5581}
5582
5583/// Emit reduction operation for each element of array (required for
5584/// array sections) LHS op = RHS.
5585/// \param Type Type of array.
5586/// \param LHSVar Variable on the left side of the reduction operation
5587/// (references element of array in original variable).
5588/// \param RHSVar Variable on the right side of the reduction operation
5589/// (references element of array in original variable).
5590/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5591/// RHSVar.
5592static void EmitOMPAggregateReduction(
5593 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5594 const VarDecl *RHSVar,
5595 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5596 const Expr *, const Expr *)> &RedOpGen,
5597 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5598 const Expr *UpExpr = nullptr) {
5599 // Perform element-by-element initialization.
5600 QualType ElementTy;
5601 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5602 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5603
5604 // Drill down to the base element type on both arrays.
5605 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5606 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5607
5608 llvm::Value *RHSBegin = RHSAddr.getPointer();
5609 llvm::Value *LHSBegin = LHSAddr.getPointer();
5610 // Cast from pointer to array type to pointer to single element.
5611 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5612 // The basic structure here is a while-do loop.
5613 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5614 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5615 llvm::Value *IsEmpty =
5616 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5617 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5618
5619 // Enter the loop body, making that address the current address.
5620 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5621 CGF.EmitBlock(BodyBB);
5622
5623 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5624
5625 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5626 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5627 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5628 Address RHSElementCurrent =
5629 Address(RHSElementPHI,
5630 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5631
5632 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5633 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5634 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5635 Address LHSElementCurrent =
5636 Address(LHSElementPHI,
5637 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5638
5639 // Emit copy.
5640 CodeGenFunction::OMPPrivateScope Scope(CGF);
5641 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5642 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5643 Scope.Privatize();
5644 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5645 Scope.ForceCleanup();
5646
5647 // Shift the address forward by one element.
5648 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5649 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5650 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5651 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5652 // Check whether we've reached the end.
5653 llvm::Value *Done =
5654 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5655 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5656 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5657 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5658
5659 // Done.
5660 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5661}
5662
5663/// Emit reduction combiner. If the combiner is a simple expression emit it as
5664/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5665/// UDR combiner function.
5666static void emitReductionCombiner(CodeGenFunction &CGF,
5667 const Expr *ReductionOp) {
5668 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5669 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5670 if (const auto *DRE =
5671 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5672 if (const auto *DRD =
5673 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5674 std::pair<llvm::Function *, llvm::Function *> Reduction =
5675 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5676 RValue Func = RValue::get(Reduction.first);
5677 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5678 CGF.EmitIgnoredExpr(ReductionOp);
5679 return;
5680 }
5681 CGF.EmitIgnoredExpr(ReductionOp);
5682}
5683
5684llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5685 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5686 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5687 ArrayRef<const Expr *> ReductionOps) {
5688 ASTContext &C = CGM.getContext();
5689
5690 // void reduction_func(void *LHSArg, void *RHSArg);
5691 FunctionArgList Args;
5692 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5693 ImplicitParamDecl::Other);
5694 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5695 ImplicitParamDecl::Other);
5696 Args.push_back(&LHSArg);
5697 Args.push_back(&RHSArg);
5698 const auto &CGFI =
5699 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5700 std::string Name = getName({"omp", "reduction", "reduction_func"});
5701 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5702 llvm::GlobalValue::InternalLinkage, Name,
5703 &CGM.getModule());
5704 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5705 Fn->setDoesNotRecurse();
5706 CodeGenFunction CGF(CGM);
5707 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5708
5709 // Dst = (void*[n])(LHSArg);
5710 // Src = (void*[n])(RHSArg);
5711 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5712 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5713 ArgsType), CGF.getPointerAlign());
5714 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5715 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5716 ArgsType), CGF.getPointerAlign());
5717
5718 // ...
5719 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5720 // ...
5721 CodeGenFunction::OMPPrivateScope Scope(CGF);
5722 auto IPriv = Privates.begin();
5723 unsigned Idx = 0;
5724 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5725 const auto *RHSVar =
5726 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5727 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5728 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5729 });
5730 const auto *LHSVar =
5731 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5732 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5733 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5734 });
5735 QualType PrivTy = (*IPriv)->getType();
5736 if (PrivTy->isVariablyModifiedType()) {
5737 // Get array size and emit VLA type.
5738 ++Idx;
5739 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5740 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5741 const VariableArrayType *VLA =
5742 CGF.getContext().getAsVariableArrayType(PrivTy);
5743 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5744 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5745 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5746 CGF.EmitVariablyModifiedType(PrivTy);
5747 }
5748 }
5749 Scope.Privatize();
5750 IPriv = Privates.begin();
5751 auto ILHS = LHSExprs.begin();
5752 auto IRHS = RHSExprs.begin();
5753 for (const Expr *E : ReductionOps) {
5754 if ((*IPriv)->getType()->isArrayType()) {
5755 // Emit reduction for array section.
5756 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5757 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5758 EmitOMPAggregateReduction(
5759 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5760 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5761 emitReductionCombiner(CGF, E);
5762 });
5763 } else {
5764 // Emit reduction for array subscript or single variable.
5765 emitReductionCombiner(CGF, E);
5766 }
5767 ++IPriv;
5768 ++ILHS;
5769 ++IRHS;
5770 }
5771 Scope.ForceCleanup();
5772 CGF.FinishFunction();
5773 return Fn;
5774}
5775
5776void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5777 const Expr *ReductionOp,
5778 const Expr *PrivateRef,
5779 const DeclRefExpr *LHS,
5780 const DeclRefExpr *RHS) {
5781 if (PrivateRef->getType()->isArrayType()) {
5782 // Emit reduction for array section.
5783 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5784 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5785 EmitOMPAggregateReduction(
5786 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5787 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5788 emitReductionCombiner(CGF, ReductionOp);
5789 });
5790 } else {
5791 // Emit reduction for array subscript or single variable.
5792 emitReductionCombiner(CGF, ReductionOp);
5793 }
5794}
5795
5796void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5797 ArrayRef<const Expr *> Privates,
5798 ArrayRef<const Expr *> LHSExprs,
5799 ArrayRef<const Expr *> RHSExprs,
5800 ArrayRef<const Expr *> ReductionOps,
5801 ReductionOptionsTy Options) {
5802 if (!CGF.HaveInsertPoint())
5803 return;
5804
5805 bool WithNowait = Options.WithNowait;
5806 bool SimpleReduction = Options.SimpleReduction;
5807
5808 // Next code should be emitted for reduction:
5809 //
5810 // static kmp_critical_name lock = { 0 };
5811 //
5812 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5813 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5814 // ...
5815 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5816 // *(Type<n>-1*)rhs[<n>-1]);
5817 // }
5818 //
5819 // ...
5820 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5821 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5822 // RedList, reduce_func, &<lock>)) {
5823 // case 1:
5824 // ...
5825 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5826 // ...
5827 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5828 // break;
5829 // case 2:
5830 // ...
5831 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5832 // ...
5833 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5834 // break;
5835 // default:;
5836 // }
5837 //
5838 // if SimpleReduction is true, only the next code is generated:
5839 // ...
5840 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5841 // ...
5842
5843 ASTContext &C = CGM.getContext();
5844
5845 if (SimpleReduction) {
5846 CodeGenFunction::RunCleanupsScope Scope(CGF);
5847 auto IPriv = Privates.begin();
5848 auto ILHS = LHSExprs.begin();
5849 auto IRHS = RHSExprs.begin();
5850 for (const Expr *E : ReductionOps) {
5851 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5852 cast<DeclRefExpr>(*IRHS));
5853 ++IPriv;
5854 ++ILHS;
5855 ++IRHS;
5856 }
5857 return;
5858 }
5859
5860 // 1. Build a list of reduction variables.
5861 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5862 auto Size = RHSExprs.size();
5863 for (const Expr *E : Privates) {
5864 if (E->getType()->isVariablyModifiedType())
5865 // Reserve place for array size.
5866 ++Size;
5867 }
5868 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5869 QualType ReductionArrayTy =
5870 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
5871 /*IndexTypeQuals=*/0);
5872 Address ReductionList =
5873 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5874 auto IPriv = Privates.begin();
5875 unsigned Idx = 0;
5876 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5877 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5878 CGF.Builder.CreateStore(
5879 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5880 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5881 Elem);
5882 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5883 // Store array size.
5884 ++Idx;
5885 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5886 llvm::Value *Size = CGF.Builder.CreateIntCast(
5887 CGF.getVLASize(
5888 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5889 .NumElts,
5890 CGF.SizeTy, /*isSigned=*/false);
5891 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5892 Elem);
5893 }
5894 }
5895
5896 // 2. Emit reduce_func().
5897 llvm::Function *ReductionFn = emitReductionFunction(
5898 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5899 LHSExprs, RHSExprs, ReductionOps);
5900
5901 // 3. Create static kmp_critical_name lock = { 0 };
5902 std::string Name = getName({"reduction"});
5903 llvm::Value *Lock = getCriticalRegionLock(Name);
5904
5905 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5906 // RedList, reduce_func, &<lock>);
5907 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5908 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5909 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5910 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5911 ReductionList.getPointer(), CGF.VoidPtrTy);
5912 llvm::Value *Args[] = {
5913 IdentTLoc, // ident_t *<loc>
5914 ThreadId, // i32 <gtid>
5915 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5916 ReductionArrayTySize, // size_type sizeof(RedList)
5917 RL, // void *RedList
5918 ReductionFn, // void (*) (void *, void *) <reduce_func>
5919 Lock // kmp_critical_name *&<lock>
5920 };
5921 llvm::Value *Res = CGF.EmitRuntimeCall(
5922 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5923 : OMPRTL__kmpc_reduce),
5924 Args);
5925
5926 // 5. Build switch(res)
5927 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5928 llvm::SwitchInst *SwInst =
5929 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5930
5931 // 6. Build case 1:
5932 // ...
5933 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5934 // ...
5935 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5936 // break;
5937 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5938 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5939 CGF.EmitBlock(Case1BB);
5940
5941 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5942 llvm::Value *EndArgs[] = {
5943 IdentTLoc, // ident_t *<loc>
5944 ThreadId, // i32 <gtid>
5945 Lock // kmp_critical_name *&<lock>
5946 };
5947 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5948 CodeGenFunction &CGF, PrePostActionTy &Action) {
5949 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5950 auto IPriv = Privates.begin();
5951 auto ILHS = LHSExprs.begin();
5952 auto IRHS = RHSExprs.begin();
5953 for (const Expr *E : ReductionOps) {
5954 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5955 cast<DeclRefExpr>(*IRHS));
5956 ++IPriv;
5957 ++ILHS;
5958 ++IRHS;
5959 }
5960 };
5961 RegionCodeGenTy RCG(CodeGen);
5962 CommonActionTy Action(
5963 nullptr, llvm::None,
5964 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5965 : OMPRTL__kmpc_end_reduce),
5966 EndArgs);
5967 RCG.setAction(Action);
5968 RCG(CGF);
5969
5970 CGF.EmitBranch(DefaultBB);
5971
5972 // 7. Build case 2:
5973 // ...
5974 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5975 // ...
5976 // break;
5977 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5978 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5979 CGF.EmitBlock(Case2BB);
5980
5981 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5982 CodeGenFunction &CGF, PrePostActionTy &Action) {
5983 auto ILHS = LHSExprs.begin();
5984 auto IRHS = RHSExprs.begin();
5985 auto IPriv = Privates.begin();
5986 for (const Expr *E : ReductionOps) {
5987 const Expr *XExpr = nullptr;
5988 const Expr *EExpr = nullptr;
5989 const Expr *UpExpr = nullptr;
5990 BinaryOperatorKind BO = BO_Comma;
5991 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5992 if (BO->getOpcode() == BO_Assign) {
5993 XExpr = BO->getLHS();
5994 UpExpr = BO->getRHS();
5995 }
5996 }
5997 // Try to emit update expression as a simple atomic.
5998 const Expr *RHSExpr = UpExpr;
5999 if (RHSExpr) {
6000 // Analyze RHS part of the whole expression.
6001 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
6002 RHSExpr->IgnoreParenImpCasts())) {
6003 // If this is a conditional operator, analyze its condition for
6004 // min/max reduction operator.
6005 RHSExpr = ACO->getCond();
6006 }
6007 if (const auto *BORHS =
6008 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
6009 EExpr = BORHS->getRHS();
6010 BO = BORHS->getOpcode();
6011 }
6012 }
6013 if (XExpr) {
6014 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
6015 auto &&AtomicRedGen = [BO, VD,
6016 Loc](CodeGenFunction &CGF, const Expr *XExpr,
6017 const Expr *EExpr, const Expr *UpExpr) {
6018 LValue X = CGF.EmitLValue(XExpr);
6019 RValue E;
6020 if (EExpr)
6021 E = CGF.EmitAnyExpr(EExpr);
6022 CGF.EmitOMPAtomicSimpleUpdateExpr(
6023 X, E, BO, /*IsXLHSInRHSPart=*/true,
6024 llvm::AtomicOrdering::Monotonic, Loc,
6025 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
6026 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6027 PrivateScope.addPrivate(
6028 VD, [&CGF, VD, XRValue, Loc]() {
6029 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
6030 CGF.emitOMPSimpleStore(
6031 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
6032 VD->getType().getNonReferenceType(), Loc);
6033 return LHSTemp;
6034 });
6035 (void)PrivateScope.Privatize();
6036 return CGF.EmitAnyExpr(UpExpr);
6037 });
6038 };
6039 if ((*IPriv)->getType()->isArrayType()) {
6040 // Emit atomic reduction for array section.
6041 const auto *RHSVar =
6042 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
6043 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
6044 AtomicRedGen, XExpr, EExpr, UpExpr);
6045 } else {
6046 // Emit atomic reduction for array subscript or single variable.
6047 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
6048 }
6049 } else {
6050 // Emit as a critical region.
6051 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
6052 const Expr *, const Expr *) {
6053 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6054 std::string Name = RT.getName({"atomic_reduction"});
6055 RT.emitCriticalRegion(
6056 CGF, Name,
6057 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
6058 Action.Enter(CGF);
6059 emitReductionCombiner(CGF, E);
6060 },
6061 Loc);
6062 };
6063 if ((*IPriv)->getType()->isArrayType()) {
6064 const auto *LHSVar =
6065 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
6066 const auto *RHSVar =
6067 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
6068 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
6069 CritRedGen);
6070 } else {
6071 CritRedGen(CGF, nullptr, nullptr, nullptr);
6072 }
6073 }
6074 ++ILHS;
6075 ++IRHS;
6076 ++IPriv;
6077 }
6078 };
6079 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
6080 if (!WithNowait) {
6081 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
6082 llvm::Value *EndArgs[] = {
6083 IdentTLoc, // ident_t *<loc>
6084 ThreadId, // i32 <gtid>
6085 Lock // kmp_critical_name *&<lock>
6086 };
6087 CommonActionTy Action(nullptr, llvm::None,
6088 createRuntimeFunction(OMPRTL__kmpc_end_reduce),
6089 EndArgs);
6090 AtomicRCG.setAction(Action);
6091 AtomicRCG(CGF);
6092 } else {
6093 AtomicRCG(CGF);
6094 }
6095
6096 CGF.EmitBranch(DefaultBB);
6097 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
6098}
6099
6100/// Generates unique name for artificial threadprivate variables.
6101/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
6102static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
6103 const Expr *Ref) {
6104 SmallString<256> Buffer;
6105 llvm::raw_svector_ostream Out(Buffer);
6106 const clang::DeclRefExpr *DE;
6107 const VarDecl *D = ::getBaseDecl(Ref, DE);
6108 if (!D)
6109 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
6110 D = D->getCanonicalDecl();
6111 std::string Name = CGM.getOpenMPRuntime().getName(
6112 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
6113 Out << Prefix << Name << "_"
6114 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
6115 return Out.str();
6116}
6117
6118/// Emits reduction initializer function:
6119/// \code
6120/// void @.red_init(void* %arg) {
6121/// %0 = bitcast void* %arg to <type>*
6122/// store <type> <init>, <type>* %0
6123/// ret void
6124/// }
6125/// \endcode
6126static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
6127 SourceLocation Loc,
6128 ReductionCodeGen &RCG, unsigned N) {
6129 ASTContext &C = CGM.getContext();
6130 FunctionArgList Args;
6131 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6132 ImplicitParamDecl::Other);
6133 Args.emplace_back(&Param);
6134 const auto &FnInfo =
6135 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6136 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6137 std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});
6138 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6139 Name, &CGM.getModule());
6140 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6141 Fn->setDoesNotRecurse();
6142 CodeGenFunction CGF(CGM);
6143 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6144 Address PrivateAddr = CGF.EmitLoadOfPointer(
6145 CGF.GetAddrOfLocalVar(&Param),
6146 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6147 llvm::Value *Size = nullptr;
6148 // If the size of the reduction item is non-constant, load it from global
6149 // threadprivate variable.
6150 if (RCG.getSizes(N).second) {
6151 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6152 CGF, CGM.getContext().getSizeType(),
6153 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6154 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6155 CGM.getContext().getSizeType(), Loc);
6156 }
6157 RCG.emitAggregateType(CGF, N, Size);
6158 LValue SharedLVal;
6159 // If initializer uses initializer from declare reduction construct, emit a
6160 // pointer to the address of the original reduction item (reuired by reduction
6161 // initializer)
6162 if (RCG.usesReductionInitializer(N)) {
6163 Address SharedAddr =
6164 CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6165 CGF, CGM.getContext().VoidPtrTy,
6166 generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6167 SharedAddr = CGF.EmitLoadOfPointer(
6168 SharedAddr,
6169 CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
6170 SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
6171 } else {
6172 SharedLVal = CGF.MakeNaturalAlignAddrLValue(
6173 llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
6174 CGM.getContext().VoidPtrTy);
6175 }
6176 // Emit the initializer:
6177 // %0 = bitcast void* %arg to <type>*
6178 // store <type> <init>, <type>* %0
6179 RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal,
6180 [](CodeGenFunction &) { return false; });
6181 CGF.FinishFunction();
6182 return Fn;
6183}
6184
6185/// Emits reduction combiner function:
6186/// \code
6187/// void @.red_comb(void* %arg0, void* %arg1) {
6188/// %lhs = bitcast void* %arg0 to <type>*
6189/// %rhs = bitcast void* %arg1 to <type>*
6190/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
6191/// store <type> %2, <type>* %lhs
6192/// ret void
6193/// }
6194/// \endcode
6195static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
6196 SourceLocation Loc,
6197 ReductionCodeGen &RCG, unsigned N,
6198 const Expr *ReductionOp,
6199 const Expr *LHS, const Expr *RHS,
6200 const Expr *PrivateRef) {
6201 ASTContext &C = CGM.getContext();
6202 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
6203 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
6204 FunctionArgList Args;
6205 ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
6206 C.VoidPtrTy, ImplicitParamDecl::Other);
6207 ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6208 ImplicitParamDecl::Other);
6209 Args.emplace_back(&ParamInOut);
6210 Args.emplace_back(&ParamIn);
6211 const auto &FnInfo =
6212 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6213 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6214 std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});
6215 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6216 Name, &CGM.getModule());
6217 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6218 Fn->setDoesNotRecurse();
6219 CodeGenFunction CGF(CGM);
6220 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6221 llvm::Value *Size = nullptr;
6222 // If the size of the reduction item is non-constant, load it from global
6223 // threadprivate variable.
6224 if (RCG.getSizes(N).second) {
6225 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6226 CGF, CGM.getContext().getSizeType(),
6227 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6228 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6229 CGM.getContext().getSizeType(), Loc);
6230 }
6231 RCG.emitAggregateType(CGF, N, Size);
6232 // Remap lhs and rhs variables to the addresses of the function arguments.
6233 // %lhs = bitcast void* %arg0 to <type>*
6234 // %rhs = bitcast void* %arg1 to <type>*
6235 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6236 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
6237 // Pull out the pointer to the variable.
6238 Address PtrAddr = CGF.EmitLoadOfPointer(
6239 CGF.GetAddrOfLocalVar(&ParamInOut),
6240 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6241 return CGF.Builder.CreateElementBitCast(
6242 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
6243 });
6244 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
6245 // Pull out the pointer to the variable.
6246 Address PtrAddr = CGF.EmitLoadOfPointer(
6247 CGF.GetAddrOfLocalVar(&ParamIn),
6248 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6249 return CGF.Builder.CreateElementBitCast(
6250 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
6251 });
6252 PrivateScope.Privatize();
6253 // Emit the combiner body:
6254 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6255 // store <type> %2, <type>* %lhs
6256 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
6257 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6258 cast<DeclRefExpr>(RHS));
6259 CGF.FinishFunction();
6260 return Fn;
6261}
6262
6263/// Emits reduction finalizer function:
6264/// \code
6265/// void @.red_fini(void* %arg) {
6266/// %0 = bitcast void* %arg to <type>*
6267/// <destroy>(<type>* %0)
6268/// ret void
6269/// }
6270/// \endcode
6271static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6272 SourceLocation Loc,
6273 ReductionCodeGen &RCG, unsigned N) {
6274 if (!RCG.needCleanups(N))
6275 return nullptr;
6276 ASTContext &C = CGM.getContext();
6277 FunctionArgList Args;
6278 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6279 ImplicitParamDecl::Other);
6280 Args.emplace_back(&Param);
6281 const auto &FnInfo =
6282 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6283 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6284 std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});
6285 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6286 Name, &CGM.getModule());
6287 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6288 Fn->setDoesNotRecurse();
6289 CodeGenFunction CGF(CGM);
6290 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6291 Address PrivateAddr = CGF.EmitLoadOfPointer(
6292 CGF.GetAddrOfLocalVar(&Param),
6293 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6294 llvm::Value *Size = nullptr;
6295 // If the size of the reduction item is non-constant, load it from global
6296 // threadprivate variable.
6297 if (RCG.getSizes(N).second) {
6298 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6299 CGF, CGM.getContext().getSizeType(),
6300 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6301 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6302 CGM.getContext().getSizeType(), Loc);
6303 }
6304 RCG.emitAggregateType(CGF, N, Size);
6305 // Emit the finalizer body:
6306 // <destroy>(<type>* %0)
6307 RCG.emitCleanups(CGF, N, PrivateAddr);
6308 CGF.FinishFunction();
6309 return Fn;
6310}
6311
6312llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
6313 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
6314 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6315 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6316 return nullptr;
6317
6318 // Build typedef struct:
6319 // kmp_task_red_input {
6320 // void *reduce_shar; // shared reduction item
6321 // size_t reduce_size; // size of data item
6322 // void *reduce_init; // data initialization routine
6323 // void *reduce_fini; // data finalization routine
6324 // void *reduce_comb; // data combiner routine
6325 // kmp_task_red_flags_t flags; // flags for additional info from compiler
6326 // } kmp_task_red_input_t;
6327 ASTContext &C = CGM.getContext();
6328 RecordDecl *RD = C.buildImplicitRecord("kmp_task_red_input_t");
6329 RD->startDefinition();
6330 const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6331 const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
6332 const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6333 const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6334 const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6335 const FieldDecl *FlagsFD = addFieldToRecordDecl(
6336 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6337 RD->completeDefinition();
6338 QualType RDType = C.getRecordType(RD);
6339 unsigned Size = Data.ReductionVars.size();
6340 llvm::APInt ArraySize(/*numBits=*/64, Size);
6341 QualType ArrayRDType = C.getConstantArrayType(
6342 RDType, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0);
6343 // kmp_task_red_input_t .rd_input.[Size];
6344 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
6345 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies,
6346 Data.ReductionOps);
6347 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6348 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6349 llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
6350 llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
6351 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6352 TaskRedInput.getPointer(), Idxs,
6353 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6354 ".rd_input.gep.");
6355 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
6356 // ElemLVal.reduce_shar = &Shareds[Cnt];
6357 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
6358 RCG.emitSharedLValue(CGF, Cnt);
6359 llvm::Value *CastedShared =
6360 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer());
6361 CGF.EmitStoreOfScalar(CastedShared, SharedLVal);
6362 RCG.emitAggregateType(CGF, Cnt);
6363 llvm::Value *SizeValInChars;
6364 llvm::Value *SizeVal;
6365 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
6366 // We use delayed creation/initialization for VLAs, array sections and
6367 // custom reduction initializations. It is required because runtime does not
6368 // provide the way to pass the sizes of VLAs/array sections to
6369 // initializer/combiner/finalizer functions and does not pass the pointer to
6370 // original reduction item to the initializer. Instead threadprivate global
6371 // variables are used to store these values and use them in the functions.
6372 bool DelayedCreation = !!SizeVal;
6373 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
6374 /*isSigned=*/false);
6375 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
6376 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
6377 // ElemLVal.reduce_init = init;
6378 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
6379 llvm::Value *InitAddr =
6380 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt));
6381 CGF.EmitStoreOfScalar(InitAddr, InitLVal);
6382 DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
6383 // ElemLVal.reduce_fini = fini;
6384 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
6385 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
6386 llvm::Value *FiniAddr = Fini
6387 ? CGF.EmitCastToVoidPtr(Fini)
6388 : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
6389 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
6390 // ElemLVal.reduce_comb = comb;
6391 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
6392 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction(
6393 CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
6394 RHSExprs[Cnt], Data.ReductionCopies[Cnt]));
6395 CGF.EmitStoreOfScalar(CombAddr, CombLVal);
6396 // ElemLVal.flags = 0;
6397 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
6398 if (DelayedCreation) {
6399 CGF.EmitStoreOfScalar(
6400 llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true),
6401 FlagsLVal);
6402 } else
6403 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
6404 }
6405 // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void
6406 // *data);
6407 llvm::Value *Args[] = {
6408 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6409 /*isSigned=*/true),
6410 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6411 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
6412 CGM.VoidPtrTy)};
6413 return CGF.EmitRuntimeCall(
6414 createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args);
6415}
6416
6417void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6418 SourceLocation Loc,
6419 ReductionCodeGen &RCG,
6420 unsigned N) {
6421 auto Sizes = RCG.getSizes(N);
6422 // Emit threadprivate global variable if the type is non-constant
6423 // (Sizes.second = nullptr).
6424 if (Sizes.second) {
6425 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
6426 /*isSigned=*/false);
6427 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6428 CGF, CGM.getContext().getSizeType(),
6429 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6430 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
6431 }
6432 // Store address of the original reduction item if custom initializer is used.
6433 if (RCG.usesReductionInitializer(N)) {
6434 Address SharedAddr = getAddrOfArtificialThreadPrivate(
6435 CGF, CGM.getContext().VoidPtrTy,
6436 generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6437 CGF.Builder.CreateStore(
6438 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6439 RCG.getSharedLValue(N).getPointer(), CGM.VoidPtrTy),
6440 SharedAddr, /*IsVolatile=*/false);
6441 }
6442}
6443
6444Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6445 SourceLocation Loc,
6446 llvm::Value *ReductionsPtr,
6447 LValue SharedLVal) {
6448 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6449 // *d);
6450 llvm::Value *Args[] = {
6451 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6452 /*isSigned=*/true),
6453 ReductionsPtr,
6454 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(SharedLVal.getPointer(),
6455 CGM.VoidPtrTy)};
6456 return Address(
6457 CGF.EmitRuntimeCall(
6458 createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args),
6459 SharedLVal.getAlignment());
6460}
6461
6462void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
6463 SourceLocation Loc) {
6464 if (!CGF.HaveInsertPoint())
6465 return;
6466 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6467 // global_tid);
6468 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
6469 // Ignore return result until untied tasks are supported.
6470 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
6471 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
6472 Region->emitUntiedSwitch(CGF);
6473}
6474
6475void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6476 OpenMPDirectiveKind InnerKind,
6477 const RegionCodeGenTy &CodeGen,
6478 bool HasCancel) {
6479 if (!CGF.HaveInsertPoint())
6480 return;
6481 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6482 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6483}
6484
6485namespace {
6486enum RTCancelKind {
6487 CancelNoreq = 0,
6488 CancelParallel = 1,
6489 CancelLoop = 2,
6490 CancelSections = 3,
6491 CancelTaskgroup = 4
6492};
6493} // anonymous namespace
6494
6495static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6496 RTCancelKind CancelKind = CancelNoreq;
6497 if (CancelRegion == OMPD_parallel)
6498 CancelKind = CancelParallel;
6499 else if (CancelRegion == OMPD_for)
6500 CancelKind = CancelLoop;
6501 else if (CancelRegion == OMPD_sections)
6502 CancelKind = CancelSections;
6503 else {
6504 assert(CancelRegion == OMPD_taskgroup)((CancelRegion == OMPD_taskgroup) ? static_cast<void> (
0) : __assert_fail ("CancelRegion == OMPD_taskgroup", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6504, __PRETTY_FUNCTION__))
;
6505 CancelKind = CancelTaskgroup;
6506 }
6507 return CancelKind;
6508}
6509
6510void CGOpenMPRuntime::emitCancellationPointCall(
6511 CodeGenFunction &CGF, SourceLocation Loc,
6512 OpenMPDirectiveKind CancelRegion) {
6513 if (!CGF.HaveInsertPoint())
6514 return;
6515 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6516 // global_tid, kmp_int32 cncl_kind);
6517 if (auto *OMPRegionInfo =
6518 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6519 // For 'cancellation point taskgroup', the task region info may not have a
6520 // cancel. This may instead happen in another adjacent task.
6521 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6522 llvm::Value *Args[] = {
6523 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6524 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6525 // Ignore return result until untied tasks are supported.
6526 llvm::Value *Result = CGF.EmitRuntimeCall(
6527 createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
6528 // if (__kmpc_cancellationpoint()) {
6529 // exit from construct;
6530 // }
6531 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6532 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6533 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6534 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6535 CGF.EmitBlock(ExitBB);
6536 // exit from construct;
6537 CodeGenFunction::JumpDest CancelDest =
6538 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6539 CGF.EmitBranchThroughCleanup(CancelDest);
6540 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6541 }
6542 }
6543}
6544
6545void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6546 const Expr *IfCond,
6547 OpenMPDirectiveKind CancelRegion) {
6548 if (!CGF.HaveInsertPoint())
6549 return;
6550 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6551 // kmp_int32 cncl_kind);
6552 if (auto *OMPRegionInfo =
6553 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6554 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
6555 PrePostActionTy &) {
6556 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6557 llvm::Value *Args[] = {
6558 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6559 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6560 // Ignore return result until untied tasks are supported.
6561 llvm::Value *Result = CGF.EmitRuntimeCall(
6562 RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
6563 // if (__kmpc_cancel()) {
6564 // exit from construct;
6565 // }
6566 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6567 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6568 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6569 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6570 CGF.EmitBlock(ExitBB);
6571 // exit from construct;
6572 CodeGenFunction::JumpDest CancelDest =
6573 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6574 CGF.EmitBranchThroughCleanup(CancelDest);
6575 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6576 };
6577 if (IfCond) {
6578 emitOMPIfClause(CGF, IfCond, ThenGen,
6579 [](CodeGenFunction &, PrePostActionTy &) {});
6580 } else {
6581 RegionCodeGenTy ThenRCG(ThenGen);
6582 ThenRCG(CGF);
6583 }
6584 }
6585}
6586
6587void CGOpenMPRuntime::emitTargetOutlinedFunction(
6588 const OMPExecutableDirective &D, StringRef ParentName,
6589 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6590 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6591 assert(!ParentName.empty() && "Invalid target region parent name!")((!ParentName.empty() && "Invalid target region parent name!"
) ? static_cast<void> (0) : __assert_fail ("!ParentName.empty() && \"Invalid target region parent name!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6591, __PRETTY_FUNCTION__))
;
6592 HasEmittedTargetRegion = true;
6593 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6594 IsOffloadEntry, CodeGen);
6595}
6596
6597void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6598 const OMPExecutableDirective &D, StringRef ParentName,
6599 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6600 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6601 // Create a unique name for the entry function using the source location
6602 // information of the current target region. The name will be something like:
6603 //
6604 // __omp_offloading_DD_FFFF_PP_lBB
6605 //
6606 // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
6607 // mangled name of the function that encloses the target region and BB is the
6608 // line number of the target region.
6609
6610 unsigned DeviceID;
6611 unsigned FileID;
6612 unsigned Line;
6613 getTargetEntryUniqueInfo(CGM.getContext(), D.getBeginLoc(), DeviceID, FileID,
6614 Line);
6615 SmallString<64> EntryFnName;
6616 {
6617 llvm::raw_svector_ostream OS(EntryFnName);
6618 OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
6619 << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
6620 }
6621
6622 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6623
6624 CodeGenFunction CGF(CGM, true);
6625 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6626 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6627
6628 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
6629
6630 // If this target outline function is not an offload entry, we don't need to
6631 // register it.
6632 if (!IsOffloadEntry)
6633 return;
6634
6635 // The target region ID is used by the runtime library to identify the current
6636 // target region, so it only has to be unique and not necessarily point to
6637 // anything. It could be the pointer to the outlined function that implements
6638 // the target region, but we aren't using that so that the compiler doesn't
6639 // need to keep that, and could therefore inline the host function if proven
6640 // worthwhile during optimization. In the other hand, if emitting code for the
6641 // device, the ID has to be the function address so that it can retrieved from
6642 // the offloading entry and launched by the runtime library. We also mark the
6643 // outlined function to have external linkage in case we are emitting code for
6644 // the device, because these functions will be entry points to the device.
6645
6646 if (CGM.getLangOpts().OpenMPIsDevice) {
6647 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
6648 OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6649 OutlinedFn->setDSOLocal(false);
6650 } else {
6651 std::string Name = getName({EntryFnName, "region_id"});
6652 OutlinedFnID = new llvm::GlobalVariable(
6653 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
6654 llvm::GlobalValue::WeakAnyLinkage,
6655 llvm::Constant::getNullValue(CGM.Int8Ty), Name);
6656 }
6657
6658 // Register the information for the entry associated with this target region.
6659 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
6660 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6661 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion);
6662}
6663
6664/// Checks if the expression is constant or does not have non-trivial function
6665/// calls.
6666static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6667 // We can skip constant expressions.
6668 // We can skip expressions with trivial calls or simple expressions.
6669 return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) ||
6670 !E->hasNonTrivialCall(Ctx)) &&
6671 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6672}
6673
6674const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6675 const Stmt *Body) {
6676 const Stmt *Child = Body->IgnoreContainers();
6677 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {
6678 Child = nullptr;
6679 for (const Stmt *S : C->body()) {
6680 if (const auto *E = dyn_cast<Expr>(S)) {
6681 if (isTrivial(Ctx, E))
6682 continue;
6683 }
6684 // Some of the statements can be ignored.
6685 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
6686 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
6687 continue;
6688 // Analyze declarations.
6689 if (const auto *DS = dyn_cast<DeclStmt>(S)) {
6690 if (llvm::all_of(DS->decls(), [&Ctx](const Decl *D) {
6691 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6692 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6693 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6694 isa<UsingDirectiveDecl>(D) ||
6695 isa<OMPDeclareReductionDecl>(D) ||
6696 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6697 return true;
6698 const auto *VD = dyn_cast<VarDecl>(D);
6699 if (!VD)
6700 return false;
6701 return VD->isConstexpr() ||
6702 ((VD->getType().isTrivialType(Ctx) ||
6703 VD->getType()->isReferenceType()) &&
6704 (!VD->hasInit() || isTrivial(Ctx, VD->getInit())));
6705 }))
6706 continue;
6707 }
6708 // Found multiple children - cannot get the one child only.
6709 if (Child)
6710 return nullptr;
6711 Child = S;
6712 }
6713 if (Child)
6714 Child = Child->IgnoreContainers();
6715 }
6716 return Child;
6717}
6718
6719/// Emit the number of teams for a target directive. Inspect the num_teams
6720/// clause associated with a teams construct combined or closely nested
6721/// with the target directive.
6722///
6723/// Emit a team of size one for directives such as 'target parallel' that
6724/// have no associated teams construct.
6725///
6726/// Otherwise, return nullptr.
6727static llvm::Value *
6728emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
6729 const OMPExecutableDirective &D) {
6730 assert(!CGF.getLangOpts().OpenMPIsDevice &&((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6732, __PRETTY_FUNCTION__))
6731 "Clauses associated with the teams directive expected to be emitted "((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6732, __PRETTY_FUNCTION__))
6732 "only for the host!")((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6732, __PRETTY_FUNCTION__))
;
6733 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6734 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&((isOpenMPTargetExecutionDirective(DirectiveKind) && "Expected target-based executable directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(DirectiveKind) && \"Expected target-based executable directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6735, __PRETTY_FUNCTION__))
6735 "Expected target-based executable directive.")((isOpenMPTargetExecutionDirective(DirectiveKind) && "Expected target-based executable directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(DirectiveKind) && \"Expected target-based executable directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6735, __PRETTY_FUNCTION__))
;
6736 CGBuilderTy &Bld = CGF.Builder;
6737 switch (DirectiveKind) {
6738 case OMPD_target: {
6739 const auto *CS = D.getInnermostCapturedStmt();
6740 const auto *Body =
6741 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6742 const Stmt *ChildStmt =
6743 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body);
6744 if (const auto *NestedDir =
6745 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6746 if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) {
6747 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6748 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6749 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6750 const Expr *NumTeams =
6751 NestedDir->getSingleClause<OMPNumTeamsClause>()->getNumTeams();
6752 llvm::Value *NumTeamsVal =
6753 CGF.EmitScalarExpr(NumTeams,
6754 /*IgnoreResultAssign*/ true);
6755 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6756 /*isSigned=*/true);
6757 }
6758 return Bld.getInt32(0);
6759 }
6760 if (isOpenMPParallelDirective(NestedDir->getDirectiveKind()) ||
6761 isOpenMPSimdDirective(NestedDir->getDirectiveKind()))
6762 return Bld.getInt32(1);
6763 return Bld.getInt32(0);
6764 }
6765 return nullptr;
6766 }
6767 case OMPD_target_teams:
6768 case OMPD_target_teams_distribute:
6769 case OMPD_target_teams_distribute_simd:
6770 case OMPD_target_teams_distribute_parallel_for:
6771 case OMPD_target_teams_distribute_parallel_for_simd: {
6772 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6773 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6774 const Expr *NumTeams =
6775 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams();
6776 llvm::Value *NumTeamsVal =
6777 CGF.EmitScalarExpr(NumTeams,
6778 /*IgnoreResultAssign*/ true);
6779 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6780 /*isSigned=*/true);
6781 }
6782 return Bld.getInt32(0);
6783 }
6784 case OMPD_target_parallel:
6785 case OMPD_target_parallel_for:
6786 case OMPD_target_parallel_for_simd:
6787 case OMPD_target_simd:
6788 return Bld.getInt32(1);
6789 case OMPD_parallel:
6790 case OMPD_for:
6791 case OMPD_parallel_for:
6792 case OMPD_parallel_sections:
6793 case OMPD_for_simd:
6794 case OMPD_parallel_for_simd:
6795 case OMPD_cancel:
6796 case OMPD_cancellation_point:
6797 case OMPD_ordered:
6798 case OMPD_threadprivate:
6799 case OMPD_allocate:
6800 case OMPD_task:
6801 case OMPD_simd:
6802 case OMPD_sections:
6803 case OMPD_section:
6804 case OMPD_single:
6805 case OMPD_master:
6806 case OMPD_critical:
6807 case OMPD_taskyield:
6808 case OMPD_barrier:
6809 case OMPD_taskwait:
6810 case OMPD_taskgroup:
6811 case OMPD_atomic:
6812 case OMPD_flush:
6813 case OMPD_teams:
6814 case OMPD_target_data:
6815 case OMPD_target_exit_data:
6816 case OMPD_target_enter_data:
6817 case OMPD_distribute:
6818 case OMPD_distribute_simd:
6819 case OMPD_distribute_parallel_for:
6820 case OMPD_distribute_parallel_for_simd:
6821 case OMPD_teams_distribute:
6822 case OMPD_teams_distribute_simd:
6823 case OMPD_teams_distribute_parallel_for:
6824 case OMPD_teams_distribute_parallel_for_simd:
6825 case OMPD_target_update:
6826 case OMPD_declare_simd:
6827 case OMPD_declare_variant:
6828 case OMPD_declare_target:
6829 case OMPD_end_declare_target:
6830 case OMPD_declare_reduction:
6831 case OMPD_declare_mapper:
6832 case OMPD_taskloop:
6833 case OMPD_taskloop_simd:
6834 case OMPD_requires:
6835 case OMPD_unknown:
6836 break;
6837 }
6838 llvm_unreachable("Unexpected directive kind.")::llvm::llvm_unreachable_internal("Unexpected directive kind."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6838)
;
6839}
6840
6841static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6842 llvm::Value *DefaultThreadLimitVal) {
6843 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6844 CGF.getContext(), CS->getCapturedStmt());
6845 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6846 if (isOpenMPParallelDirective(Dir->getDirectiveKind())) {
6847 llvm::Value *NumThreads = nullptr;
6848 llvm::Value *CondVal = nullptr;
6849 // Handle if clause. If if clause present, the number of threads is
6850 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6851 if (Dir->hasClausesOfKind<OMPIfClause>()) {
6852 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6853 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6854 const OMPIfClause *IfClause = nullptr;
6855 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6856 if (C->getNameModifier() == OMPD_unknown ||
6857 C->getNameModifier() == OMPD_parallel) {
6858 IfClause = C;
6859 break;
6860 }
6861 }
6862 if (IfClause) {
6863 const Expr *Cond = IfClause->getCondition();
6864 bool Result;
6865 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6866 if (!Result)
6867 return CGF.Builder.getInt32(1);
6868 } else {
6869 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange());
6870 if (const auto *PreInit =
6871 cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) {
6872 for (const auto *I : PreInit->decls()) {
6873 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6874 CGF.EmitVarDecl(cast<VarDecl>(*I));
6875 } else {
6876 CodeGenFunction::AutoVarEmission Emission =
6877 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6878 CGF.EmitAutoVarCleanups(Emission);
6879 }
6880 }
6881 }
6882 CondVal = CGF.EvaluateExprAsBool(Cond);
6883 }
6884 }
6885 }
6886 // Check the value of num_threads clause iff if clause was not specified
6887 // or is not evaluated to false.
6888 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6889 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6890 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6891 const auto *NumThreadsClause =
6892 Dir->getSingleClause<OMPNumThreadsClause>();
6893 CodeGenFunction::LexicalScope Scope(
6894 CGF, NumThreadsClause->getNumThreads()->getSourceRange());
6895 if (const auto *PreInit =
6896 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6897 for (const auto *I : PreInit->decls()) {
6898 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6899 CGF.EmitVarDecl(cast<VarDecl>(*I));
6900 } else {
6901 CodeGenFunction::AutoVarEmission Emission =
6902 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6903 CGF.EmitAutoVarCleanups(Emission);
6904 }
6905 }
6906 }
6907 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads());
6908 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty,
6909 /*isSigned=*/false);
6910 if (DefaultThreadLimitVal)
6911 NumThreads = CGF.Builder.CreateSelect(
6912 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads),
6913 DefaultThreadLimitVal, NumThreads);
6914 } else {
6915 NumThreads = DefaultThreadLimitVal ? DefaultThreadLimitVal
6916 : CGF.Builder.getInt32(0);
6917 }
6918 // Process condition of the if clause.
6919 if (CondVal) {
6920 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads,
6921 CGF.Builder.getInt32(1));
6922 }
6923 return NumThreads;
6924 }
6925 if (isOpenMPSimdDirective(Dir->getDirectiveKind()))
6926 return CGF.Builder.getInt32(1);
6927 return DefaultThreadLimitVal;
6928 }
6929 return DefaultThreadLimitVal ? DefaultThreadLimitVal
6930 : CGF.Builder.getInt32(0);
6931}
6932
6933/// Emit the number of threads for a target directive. Inspect the
6934/// thread_limit clause associated with a teams construct combined or closely
6935/// nested with the target directive.
6936///
6937/// Emit the num_threads clause for directives such as 'target parallel' that
6938/// have no associated teams construct.
6939///
6940/// Otherwise, return nullptr.
6941static llvm::Value *
6942emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
6943 const OMPExecutableDirective &D) {
6944 assert(!CGF.getLangOpts().OpenMPIsDevice &&((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6946, __PRETTY_FUNCTION__))
6945 "Clauses associated with the teams directive expected to be emitted "((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6946, __PRETTY_FUNCTION__))
6946 "only for the host!")((!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the teams directive expected to be emitted "
"only for the host!") ? static_cast<void> (0) : __assert_fail
("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the teams directive expected to be emitted \" \"only for the host!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6946, __PRETTY_FUNCTION__))
;
6947 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6948 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&((isOpenMPTargetExecutionDirective(DirectiveKind) && "Expected target-based executable directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(DirectiveKind) && \"Expected target-based executable directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6949, __PRETTY_FUNCTION__))
6949 "Expected target-based executable directive.")((isOpenMPTargetExecutionDirective(DirectiveKind) && "Expected target-based executable directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(DirectiveKind) && \"Expected target-based executable directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 6949, __PRETTY_FUNCTION__))
;
6950 CGBuilderTy &Bld = CGF.Builder;
6951 llvm::Value *ThreadLimitVal = nullptr;
6952 llvm::Value *NumThreadsVal = nullptr;
6953 switch (DirectiveKind) {
6954 case OMPD_target: {
6955 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6956 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6957 return NumThreads;
6958 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6959 CGF.getContext(), CS->getCapturedStmt());
6960 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6961 if (Dir->hasClausesOfKind<OMPThreadLimitClause>()) {
6962 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6963 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6964 const auto *ThreadLimitClause =
6965 Dir->getSingleClause<OMPThreadLimitClause>();
6966 CodeGenFunction::LexicalScope Scope(
6967 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6968 if (const auto *PreInit =
6969 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6970 for (const auto *I : PreInit->decls()) {
6971 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6972 CGF.EmitVarDecl(cast<VarDecl>(*I));
6973 } else {
6974 CodeGenFunction::AutoVarEmission Emission =
6975 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6976 CGF.EmitAutoVarCleanups(Emission);
6977 }
6978 }
6979 }
6980 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
6981 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
6982 ThreadLimitVal =
6983 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
6984 }
6985 if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) &&
6986 !isOpenMPDistributeDirective(Dir->getDirectiveKind())) {
6987 CS = Dir->getInnermostCapturedStmt();
6988 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6989 CGF.getContext(), CS->getCapturedStmt());
6990 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6991 }
6992 if (Dir && isOpenMPDistributeDirective(Dir->getDirectiveKind()) &&
6993 !isOpenMPSimdDirective(Dir->getDirectiveKind())) {
6994 CS = Dir->getInnermostCapturedStmt();
6995 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6996 return NumThreads;
6997 }
6998 if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind()))
6999 return Bld.getInt32(1);
7000 }
7001 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
7002 }
7003 case OMPD_target_teams: {
7004 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7005 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7006 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7007 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
7008 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
7009 ThreadLimitVal =
7010 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
7011 }
7012 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7013 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
7014 return NumThreads;
7015 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7016 CGF.getContext(), CS->getCapturedStmt());
7017 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7018 if (Dir->getDirectiveKind() == OMPD_distribute) {
7019 CS = Dir->getInnermostCapturedStmt();
7020 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
7021 return NumThreads;
7022 }
7023 }
7024 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
7025 }
7026 case OMPD_target_teams_distribute:
7027 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7028 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7029 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7030 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
7031 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
7032 ThreadLimitVal =
7033 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
7034 }
7035 return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal);
7036 case OMPD_target_parallel:
7037 case OMPD_target_parallel_for:
7038 case OMPD_target_parallel_for_simd:
7039 case OMPD_target_teams_distribute_parallel_for:
7040 case OMPD_target_teams_distribute_parallel_for_simd: {
7041 llvm::Value *CondVal = nullptr;
7042 // Handle if clause. If if clause present, the number of threads is
7043 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
7044 if (D.hasClausesOfKind<OMPIfClause>()) {
7045 const OMPIfClause *IfClause = nullptr;
7046 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
7047 if (C->getNameModifier() == OMPD_unknown ||
7048 C->getNameModifier() == OMPD_parallel) {
7049 IfClause = C;
7050 break;
7051 }
7052 }
7053 if (IfClause) {
7054 const Expr *Cond = IfClause->getCondition();
7055 bool Result;
7056 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
7057 if (!Result)
7058 return Bld.getInt32(1);
7059 } else {
7060 CodeGenFunction::RunCleanupsScope Scope(CGF);
7061 CondVal = CGF.EvaluateExprAsBool(Cond);
7062 }
7063 }
7064 }
7065 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7066 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7067 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7068 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
7069 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
7070 ThreadLimitVal =
7071 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
7072 }
7073 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
7074 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
7075 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
7076 llvm::Value *NumThreads = CGF.EmitScalarExpr(
7077 NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true);
7078 NumThreadsVal =
7079 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false);
7080 ThreadLimitVal = ThreadLimitVal
7081 ? Bld.CreateSelect(Bld.CreateICmpULT(NumThreadsVal,
7082 ThreadLimitVal),
7083 NumThreadsVal, ThreadLimitVal)
7084 : NumThreadsVal;
7085 }
7086 if (!ThreadLimitVal)
7087 ThreadLimitVal = Bld.getInt32(0);
7088 if (CondVal)
7089 return Bld.CreateSelect(CondVal, ThreadLimitVal, Bld.getInt32(1));
7090 return ThreadLimitVal;
7091 }
7092 case OMPD_target_teams_distribute_simd:
7093 case OMPD_target_simd:
7094 return Bld.getInt32(1);
7095 case OMPD_parallel:
7096 case OMPD_for:
7097 case OMPD_parallel_for:
7098 case OMPD_parallel_sections:
7099 case OMPD_for_simd:
7100 case OMPD_parallel_for_simd:
7101 case OMPD_cancel:
7102 case OMPD_cancellation_point:
7103 case OMPD_ordered:
7104 case OMPD_threadprivate:
7105 case OMPD_allocate:
7106 case OMPD_task:
7107 case OMPD_simd:
7108 case OMPD_sections:
7109 case OMPD_section:
7110 case OMPD_single:
7111 case OMPD_master:
7112 case OMPD_critical:
7113 case OMPD_taskyield:
7114 case OMPD_barrier:
7115 case OMPD_taskwait:
7116 case OMPD_taskgroup:
7117 case OMPD_atomic:
7118 case OMPD_flush:
7119 case OMPD_teams:
7120 case OMPD_target_data:
7121 case OMPD_target_exit_data:
7122 case OMPD_target_enter_data:
7123 case OMPD_distribute:
7124 case OMPD_distribute_simd:
7125 case OMPD_distribute_parallel_for:
7126 case OMPD_distribute_parallel_for_simd:
7127 case OMPD_teams_distribute:
7128 case OMPD_teams_distribute_simd:
7129 case OMPD_teams_distribute_parallel_for:
7130 case OMPD_teams_distribute_parallel_for_simd:
7131 case OMPD_target_update:
7132 case OMPD_declare_simd:
7133 case OMPD_declare_variant:
7134 case OMPD_declare_target:
7135 case OMPD_end_declare_target:
7136 case OMPD_declare_reduction:
7137 case OMPD_declare_mapper:
7138 case OMPD_taskloop:
7139 case OMPD_taskloop_simd:
7140 case OMPD_requires:
7141 case OMPD_unknown:
7142 break;
7143 }
7144 llvm_unreachable("Unsupported directive kind.")::llvm::llvm_unreachable_internal("Unsupported directive kind."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7144)
;
7145}
7146
7147namespace {
7148LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
7149
7150// Utility to handle information from clauses associated with a given
7151// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7152// It provides a convenient interface to obtain the information and generate
7153// code for that information.
7154class MappableExprsHandler {
7155public:
7156 /// Values for bit flags used to specify the mapping type for
7157 /// offloading.
7158 enum OpenMPOffloadMappingFlags : uint64_t {
7159 /// No flags
7160 OMP_MAP_NONE = 0x0,
7161 /// Allocate memory on the device and move data from host to device.
7162 OMP_MAP_TO = 0x01,
7163 /// Allocate memory on the device and move data from device to host.
7164 OMP_MAP_FROM = 0x02,
7165 /// Always perform the requested mapping action on the element, even
7166 /// if it was already mapped before.
7167 OMP_MAP_ALWAYS = 0x04,
7168 /// Delete the element from the device environment, ignoring the
7169 /// current reference count associated with the element.
7170 OMP_MAP_DELETE = 0x08,
7171 /// The element being mapped is a pointer-pointee pair; both the
7172 /// pointer and the pointee should be mapped.
7173 OMP_MAP_PTR_AND_OBJ = 0x10,
7174 /// This flags signals that the base address of an entry should be
7175 /// passed to the target kernel as an argument.
7176 OMP_MAP_TARGET_PARAM = 0x20,
7177 /// Signal that the runtime library has to return the device pointer
7178 /// in the current position for the data being mapped. Used when we have the
7179 /// use_device_ptr clause.
7180 OMP_MAP_RETURN_PARAM = 0x40,
7181 /// This flag signals that the reference being passed is a pointer to
7182 /// private data.
7183 OMP_MAP_PRIVATE = 0x80,
7184 /// Pass the element to the device by value.
7185 OMP_MAP_LITERAL = 0x100,
7186 /// Implicit map
7187 OMP_MAP_IMPLICIT = 0x200,
7188 /// Close is a hint to the runtime to allocate memory close to
7189 /// the target device.
7190 OMP_MAP_CLOSE = 0x400,
7191 /// The 16 MSBs of the flags indicate whether the entry is member of some
7192 /// struct/class.
7193 OMP_MAP_MEMBER_OF = 0xffff000000000000,
7194 LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ OMP_MAP_MEMBER_OF)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_MAP_MEMBER_OF,
7195 };
7196
7197 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7198 static unsigned getFlagMemberOffset() {
7199 unsigned Offset = 0;
7200 for (uint64_t Remain = OMP_MAP_MEMBER_OF; !(Remain & 1);
7201 Remain = Remain >> 1)
7202 Offset++;
7203 return Offset;
7204 }
7205
7206 /// Class that associates information with a base pointer to be passed to the
7207 /// runtime library.
7208 class BasePointerInfo {
7209 /// The base pointer.
7210 llvm::Value *Ptr = nullptr;
7211 /// The base declaration that refers to this device pointer, or null if
7212 /// there is none.
7213 const ValueDecl *DevPtrDecl = nullptr;
7214
7215 public:
7216 BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
7217 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
7218 llvm::Value *operator*() const { return Ptr; }
7219 const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
7220 void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
7221 };
7222
7223 using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>;
7224 using MapValuesArrayTy = SmallVector<llvm::Value *, 4>;
7225 using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>;
7226
7227 /// Map between a struct and the its lowest & highest elements which have been
7228 /// mapped.
7229 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7230 /// HE(FieldIndex, Pointer)}
7231 struct StructRangeInfoTy {
7232 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7233 0, Address::invalid()};
7234 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7235 0, Address::invalid()};
7236 Address Base = Address::invalid();
7237 };
7238
7239private:
7240 /// Kind that defines how a device pointer has to be returned.
7241 struct MapInfo {
7242 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7243 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7244 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7245 bool ReturnDevicePointer = false;
7246 bool IsImplicit = false;
7247
7248 MapInfo() = default;
7249 MapInfo(
7250 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7251 OpenMPMapClauseKind MapType,
7252 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7253 bool ReturnDevicePointer, bool IsImplicit)
7254 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7255 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
7256 };
7257
7258 /// If use_device_ptr is used on a pointer which is a struct member and there
7259 /// is no map information about it, then emission of that entry is deferred
7260 /// until the whole struct has been processed.
7261 struct DeferredDevicePtrEntryTy {
7262 const Expr *IE = nullptr;
7263 const ValueDecl *VD = nullptr;
7264
7265 DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD)
7266 : IE(IE), VD(VD) {}
7267 };
7268
7269 /// The target directive from where the mappable clauses were extracted. It
7270 /// is either a executable directive or a user-defined mapper directive.
7271 llvm::PointerUnion<const OMPExecutableDirective *,
7272 const OMPDeclareMapperDecl *>
7273 CurDir;
7274
7275 /// Function the directive is being generated for.
7276 CodeGenFunction &CGF;
7277
7278 /// Set of all first private variables in the current directive.
7279 /// bool data is set to true if the variable is implicitly marked as
7280 /// firstprivate, false otherwise.
7281 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7282
7283 /// Map between device pointer declarations and their expression components.
7284 /// The key value for declarations in 'this' is null.
7285 llvm::DenseMap<
7286 const ValueDecl *,
7287 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7288 DevPointersMap;
7289
7290 llvm::Value *getExprTypeSize(const Expr *E) const {
7291 QualType ExprTy = E->getType().getCanonicalType();
7292
7293 // Reference types are ignored for mapping purposes.
7294 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7295 ExprTy = RefTy->getPointeeType().getCanonicalType();
7296
7297 // Given that an array section is considered a built-in type, we need to
7298 // do the calculation based on the length of the section instead of relying
7299 // on CGF.getTypeSize(E->getType()).
7300 if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
7301 QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(
7302 OAE->getBase()->IgnoreParenImpCasts())
7303 .getCanonicalType();
7304
7305 // If there is no length associated with the expression and lower bound is
7306 // not specified too, that means we are using the whole length of the
7307 // base.
7308 if (!OAE->getLength() && OAE->getColonLoc().isValid() &&
7309 !OAE->getLowerBound())
7310 return CGF.getTypeSize(BaseTy);
7311
7312 llvm::Value *ElemSize;
7313 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7314 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
7315 } else {
7316 const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
7317 assert(ATy && "Expecting array type if not a pointer type.")((ATy && "Expecting array type if not a pointer type."
) ? static_cast<void> (0) : __assert_fail ("ATy && \"Expecting array type if not a pointer type.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7317, __PRETTY_FUNCTION__))
;
7318 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
7319 }
7320
7321 // If we don't have a length at this point, that is because we have an
7322 // array section with a single element.
7323 if (!OAE->getLength() && OAE->getColonLoc().isInvalid())
7324 return ElemSize;
7325
7326 if (const Expr *LenExpr = OAE->getLength()) {
7327 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
7328 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
7329 CGF.getContext().getSizeType(),
7330 LenExpr->getExprLoc());
7331 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
7332 }
7333 assert(!OAE->getLength() && OAE->getColonLoc().isValid() &&((!OAE->getLength() && OAE->getColonLoc().isValid
() && OAE->getLowerBound() && "expected array_section[lb:]."
) ? static_cast<void> (0) : __assert_fail ("!OAE->getLength() && OAE->getColonLoc().isValid() && OAE->getLowerBound() && \"expected array_section[lb:].\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7334, __PRETTY_FUNCTION__))
7334 OAE->getLowerBound() && "expected array_section[lb:].")((!OAE->getLength() && OAE->getColonLoc().isValid
() && OAE->getLowerBound() && "expected array_section[lb:]."
) ? static_cast<void> (0) : __assert_fail ("!OAE->getLength() && OAE->getColonLoc().isValid() && OAE->getLowerBound() && \"expected array_section[lb:].\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7334, __PRETTY_FUNCTION__))
;
7335 // Size = sizetype - lb * elemtype;
7336 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
7337 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
7338 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
7339 CGF.getContext().getSizeType(),
7340 OAE->getLowerBound()->getExprLoc());
7341 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
7342 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
7343 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
7344 LengthVal = CGF.Builder.CreateSelect(
7345 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
7346 return LengthVal;
7347 }
7348 return CGF.getTypeSize(ExprTy);
7349 }
7350
7351 /// Return the corresponding bits for a given map clause modifier. Add
7352 /// a flag marking the map as a pointer if requested. Add a flag marking the
7353 /// map as the first one of a series of maps that relate to the same map
7354 /// expression.
7355 OpenMPOffloadMappingFlags getMapTypeBits(
7356 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7357 bool IsImplicit, bool AddPtrFlag, bool AddIsTargetParamFlag) const {
7358 OpenMPOffloadMappingFlags Bits =
7359 IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
7360 switch (MapType) {
7361 case OMPC_MAP_alloc:
7362 case OMPC_MAP_release:
7363 // alloc and release is the default behavior in the runtime library, i.e.
7364 // if we don't pass any bits alloc/release that is what the runtime is
7365 // going to do. Therefore, we don't need to signal anything for these two
7366 // type modifiers.
7367 break;
7368 case OMPC_MAP_to:
7369 Bits |= OMP_MAP_TO;
7370 break;
7371 case OMPC_MAP_from:
7372 Bits |= OMP_MAP_FROM;
7373 break;
7374 case OMPC_MAP_tofrom:
7375 Bits |= OMP_MAP_TO | OMP_MAP_FROM;
7376 break;
7377 case OMPC_MAP_delete:
7378 Bits |= OMP_MAP_DELETE;
7379 break;
7380 case OMPC_MAP_unknown:
7381 llvm_unreachable("Unexpected map type!")::llvm::llvm_unreachable_internal("Unexpected map type!", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7381)
;
7382 }
7383 if (AddPtrFlag)
7384 Bits |= OMP_MAP_PTR_AND_OBJ;
7385 if (AddIsTargetParamFlag)
7386 Bits |= OMP_MAP_TARGET_PARAM;
7387 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always)
7388 != MapModifiers.end())
7389 Bits |= OMP_MAP_ALWAYS;
7390 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_close)
7391 != MapModifiers.end())
7392 Bits |= OMP_MAP_CLOSE;
7393 return Bits;
7394 }
7395
7396 /// Return true if the provided expression is a final array section. A
7397 /// final array section, is one whose length can't be proved to be one.
7398 bool isFinalArraySectionExpression(const Expr *E) const {
7399 const auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
7400
7401 // It is not an array section and therefore not a unity-size one.
7402 if (!OASE)
7403 return false;
7404
7405 // An array section with no colon always refer to a single element.
7406 if (OASE->getColonLoc().isInvalid())
7407 return false;
7408
7409 const Expr *Length = OASE->getLength();
7410
7411 // If we don't have a length we have to check if the array has size 1
7412 // for this dimension. Also, we should always expect a length if the
7413 // base type is pointer.
7414 if (!Length) {
7415 QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType(
7416 OASE->getBase()->IgnoreParenImpCasts())
7417 .getCanonicalType();
7418 if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
7419 return ATy->getSize().getSExtValue() != 1;
7420 // If we don't have a constant dimension length, we have to consider
7421 // the current section as having any size, so it is not necessarily
7422 // unitary. If it happen to be unity size, that's user fault.
7423 return true;
7424 }
7425
7426 // Check if the length evaluates to 1.
7427 Expr::EvalResult Result;
7428 if (!Length->EvaluateAsInt(Result, CGF.getContext()))
7429 return true; // Can have more that size 1.
7430
7431 llvm::APSInt ConstLength = Result.Val.getInt();
7432 return ConstLength.getSExtValue() != 1;
7433 }
7434
7435 /// Generate the base pointers, section pointers, sizes and map type
7436 /// bits for the provided map type, map modifier, and expression components.
7437 /// \a IsFirstComponent should be set to true if the provided set of
7438 /// components is the first associated with a capture.
7439 void generateInfoForComponentList(
7440 OpenMPMapClauseKind MapType,
7441 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7442 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7443 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
7444 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7445 StructRangeInfoTy &PartialStruct, bool IsFirstComponentList,
7446 bool IsImplicit,
7447 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7448 OverlappedElements = llvm::None) const {
7449 // The following summarizes what has to be generated for each map and the
7450 // types below. The generated information is expressed in this order:
7451 // base pointer, section pointer, size, flags
7452 // (to add to the ones that come from the map type and modifier).
7453 //
7454 // double d;
7455 // int i[100];
7456 // float *p;
7457 //
7458 // struct S1 {
7459 // int i;
7460 // float f[50];
7461 // }
7462 // struct S2 {
7463 // int i;
7464 // float f[50];
7465 // S1 s;
7466 // double *p;
7467 // struct S2 *ps;
7468 // }
7469 // S2 s;
7470 // S2 *ps;
7471 //
7472 // map(d)
7473 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
7474 //
7475 // map(i)
7476 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
7477 //
7478 // map(i[1:23])
7479 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
7480 //
7481 // map(p)
7482 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7483 //
7484 // map(p[1:24])
7485 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
7486 //
7487 // map(s)
7488 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
7489 //
7490 // map(s.i)
7491 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
7492 //
7493 // map(s.s.f)
7494 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7495 //
7496 // map(s.p)
7497 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
7498 //
7499 // map(to: s.p[:22])
7500 // &s, &(s.p), sizeof(double*), TARGET_PARAM (*)
7501 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**)
7502 // &(s.p), &(s.p[0]), 22*sizeof(double),
7503 // MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
7504 // (*) alloc space for struct members, only this is a target parameter
7505 // (**) map the pointer (nothing to be mapped in this example) (the compiler
7506 // optimizes this entry out, same in the examples below)
7507 // (***) map the pointee (map: to)
7508 //
7509 // map(s.ps)
7510 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7511 //
7512 // map(from: s.ps->s.i)
7513 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7514 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7515 // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7516 //
7517 // map(to: s.ps->ps)
7518 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7519 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7520 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | TO
7521 //
7522 // map(s.ps->ps->ps)
7523 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7524 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7525 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7526 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7527 //
7528 // map(to: s.ps->ps->s.f[:22])
7529 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7530 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7531 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7532 // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7533 //
7534 // map(ps)
7535 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7536 //
7537 // map(ps->i)
7538 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7539 //
7540 // map(ps->s.f)
7541 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7542 //
7543 // map(from: ps->p)
7544 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7545 //
7546 // map(to: ps->p[:22])
7547 // ps, &(ps->p), sizeof(double*), TARGET_PARAM
7548 // ps, &(ps->p), sizeof(double*), MEMBER_OF(1)
7549 // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO
7550 //
7551 // map(ps->ps)
7552 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7553 //
7554 // map(from: ps->ps->s.i)
7555 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7556 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7557 // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7558 //
7559 // map(from: ps->ps->ps)
7560 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7561 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7562 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7563 //
7564 // map(ps->ps->ps->ps)
7565 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7566 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7567 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7568 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7569 //
7570 // map(to: ps->ps->ps->s.f[:22])
7571 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7572 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7573 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7574 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7575 //
7576 // map(to: s.f[:22]) map(from: s.p[:33])
7577 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) +
7578 // sizeof(double*) (**), TARGET_PARAM
7579 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7580 // &s, &(s.p), sizeof(double*), MEMBER_OF(1)
7581 // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7582 // (*) allocate contiguous space needed to fit all mapped members even if
7583 // we allocate space for members not mapped (in this example,
7584 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
7585 // them as well because they fall between &s.f[0] and &s.p)
7586 //
7587 // map(from: s.f[:22]) map(to: ps->p[:33])
7588 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7589 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7590 // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*)
7591 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO
7592 // (*) the struct this entry pertains to is the 2nd element in the list of
7593 // arguments, hence MEMBER_OF(2)
7594 //
7595 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7596 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM
7597 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7598 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7599 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7600 // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*)
7601 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO
7602 // (*) the struct this entry pertains to is the 4th element in the list
7603 // of arguments, hence MEMBER_OF(4)
7604
7605 // Track if the map information being generated is the first for a capture.
7606 bool IsCaptureFirstInfo = IsFirstComponentList;
7607 // When the variable is on a declare target link or in a to clause with
7608 // unified memory, a reference is needed to hold the host/device address
7609 // of the variable.
7610 bool RequiresReference = false;
7611
7612 // Scan the components from the base to the complete expression.
7613 auto CI = Components.rbegin();
7614 auto CE = Components.rend();
7615 auto I = CI;
7616
7617 // Track if the map information being generated is the first for a list of
7618 // components.
7619 bool IsExpressionFirstInfo = true;
7620 Address BP = Address::invalid();
7621 const Expr *AssocExpr = I->getAssociatedExpression();
7622 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7623 const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7624
7625 if (isa<MemberExpr>(AssocExpr)) {
7626 // The base is the 'this' pointer. The content of the pointer is going
7627 // to be the base of the field being mapped.
7628 BP = CGF.LoadCXXThisAddress();
7629 } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7630 (OASE &&
7631 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7632 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7633 } else {
7634 // The base is the reference to the variable.
7635 // BP = &Var.
7636 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7637 if (const auto *VD =
7638 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7639 if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7640 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7641 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7642 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
7643 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
7644 RequiresReference = true;
7645 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
7646 }
7647 }
7648 }
7649
7650 // If the variable is a pointer and is being dereferenced (i.e. is not
7651 // the last component), the base has to be the pointer itself, not its
7652 // reference. References are ignored for mapping purposes.
7653 QualType Ty =
7654 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7655 if (Ty->isAnyPointerType() && std::next(I) != CE) {
7656 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7657
7658 // We do not need to generate individual map information for the
7659 // pointer, it can be associated with the combined storage.
7660 ++I;
7661 }
7662 }
7663
7664 // Track whether a component of the list should be marked as MEMBER_OF some
7665 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
7666 // in a component list should be marked as MEMBER_OF, all subsequent entries
7667 // do not belong to the base struct. E.g.
7668 // struct S2 s;
7669 // s.ps->ps->ps->f[:]
7670 // (1) (2) (3) (4)
7671 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
7672 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
7673 // is the pointee of ps(2) which is not member of struct s, so it should not
7674 // be marked as such (it is still PTR_AND_OBJ).
7675 // The variable is initialized to false so that PTR_AND_OBJ entries which
7676 // are not struct members are not considered (e.g. array of pointers to
7677 // data).
7678 bool ShouldBeMemberOf = false;
7679
7680 // Variable keeping track of whether or not we have encountered a component
7681 // in the component list which is a member expression. Useful when we have a
7682 // pointer or a final array section, in which case it is the previous
7683 // component in the list which tells us whether we have a member expression.
7684 // E.g. X.f[:]
7685 // While processing the final array section "[:]" it is "f" which tells us
7686 // whether we are dealing with a member of a declared struct.
7687 const MemberExpr *EncounteredME = nullptr;
7688
7689 for (; I != CE; ++I) {
7690 // If the current component is member of a struct (parent struct) mark it.
7691 if (!EncounteredME) {
7692 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7693 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
7694 // as MEMBER_OF the parent struct.
7695 if (EncounteredME)
7696 ShouldBeMemberOf = true;
7697 }
7698
7699 auto Next = std::next(I);
7700
7701 // We need to generate the addresses and sizes if this is the last
7702 // component, if the component is a pointer or if it is an array section
7703 // whose length can't be proved to be one. If this is a pointer, it
7704 // becomes the base address for the following components.
7705
7706 // A final array section, is one whose length can't be proved to be one.
7707 bool IsFinalArraySection =
7708 isFinalArraySectionExpression(I->getAssociatedExpression());
7709
7710 // Get information on whether the element is a pointer. Have to do a
7711 // special treatment for array sections given that they are built-in
7712 // types.
7713 const auto *OASE =
7714 dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
7715 bool IsPointer =
7716 (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE)
7717 .getCanonicalType()
7718 ->isAnyPointerType()) ||
7719 I->getAssociatedExpression()->getType()->isAnyPointerType();
7720
7721 if (Next == CE || IsPointer || IsFinalArraySection) {
7722 // If this is not the last component, we expect the pointer to be
7723 // associated with an array expression or member expression.
7724 assert((Next == CE ||(((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression
()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression
()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression
())) && "Unexpected expression") ? static_cast<void
> (0) : __assert_fail ("(Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && \"Unexpected expression\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7728, __PRETTY_FUNCTION__))
7725 isa<MemberExpr>(Next->getAssociatedExpression()) ||(((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression
()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression
()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression
())) && "Unexpected expression") ? static_cast<void
> (0) : __assert_fail ("(Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && \"Unexpected expression\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7728, __PRETTY_FUNCTION__))
7726 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||(((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression
()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression
()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression
())) && "Unexpected expression") ? static_cast<void
> (0) : __assert_fail ("(Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && \"Unexpected expression\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7728, __PRETTY_FUNCTION__))
7727 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&(((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression
()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression
()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression
())) && "Unexpected expression") ? static_cast<void
> (0) : __assert_fail ("(Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && \"Unexpected expression\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7728, __PRETTY_FUNCTION__))
7728 "Unexpected expression")(((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression
()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression
()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression
())) && "Unexpected expression") ? static_cast<void
> (0) : __assert_fail ("(Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && \"Unexpected expression\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7728, __PRETTY_FUNCTION__))
;
7729
7730 Address LB =
7731 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getAddress();
7732
7733 // If this component is a pointer inside the base struct then we don't
7734 // need to create any entry for it - it will be combined with the object
7735 // it is pointing to into a single PTR_AND_OBJ entry.
7736 bool IsMemberPointer =
7737 IsPointer && EncounteredME &&
7738 (dyn_cast<MemberExpr>(I->getAssociatedExpression()) ==
7739 EncounteredME);
7740 if (!OverlappedElements.empty()) {
7741 // Handle base element with the info for overlapped elements.
7742 assert(!PartialStruct.Base.isValid() && "The base element is set.")((!PartialStruct.Base.isValid() && "The base element is set."
) ? static_cast<void> (0) : __assert_fail ("!PartialStruct.Base.isValid() && \"The base element is set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7742, __PRETTY_FUNCTION__))
;
7743 assert(Next == CE &&((Next == CE && "Expected last element for the overlapped elements."
) ? static_cast<void> (0) : __assert_fail ("Next == CE && \"Expected last element for the overlapped elements.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7744, __PRETTY_FUNCTION__))
7744 "Expected last element for the overlapped elements.")((Next == CE && "Expected last element for the overlapped elements."
) ? static_cast<void> (0) : __assert_fail ("Next == CE && \"Expected last element for the overlapped elements.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7744, __PRETTY_FUNCTION__))
;
7745 assert(!IsPointer &&((!IsPointer && "Unexpected base element with the pointer type."
) ? static_cast<void> (0) : __assert_fail ("!IsPointer && \"Unexpected base element with the pointer type.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7746, __PRETTY_FUNCTION__))
7746 "Unexpected base element with the pointer type.")((!IsPointer && "Unexpected base element with the pointer type."
) ? static_cast<void> (0) : __assert_fail ("!IsPointer && \"Unexpected base element with the pointer type.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7746, __PRETTY_FUNCTION__))
;
7747 // Mark the whole struct as the struct that requires allocation on the
7748 // device.
7749 PartialStruct.LowestElem = {0, LB};
7750 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
7751 I->getAssociatedExpression()->getType());
7752 Address HB = CGF.Builder.CreateConstGEP(
7753 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LB,
7754 CGF.VoidPtrTy),
7755 TypeSize.getQuantity() - 1);
7756 PartialStruct.HighestElem = {
7757 std::numeric_limits<decltype(
7758 PartialStruct.HighestElem.first)>::max(),
7759 HB};
7760 PartialStruct.Base = BP;
7761 // Emit data for non-overlapped data.
7762 OpenMPOffloadMappingFlags Flags =
7763 OMP_MAP_MEMBER_OF |
7764 getMapTypeBits(MapType, MapModifiers, IsImplicit,
7765 /*AddPtrFlag=*/false,
7766 /*AddIsTargetParamFlag=*/false);
7767 LB = BP;
7768 llvm::Value *Size = nullptr;
7769 // Do bitcopy of all non-overlapped structure elements.
7770 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
7771 Component : OverlappedElements) {
7772 Address ComponentLB = Address::invalid();
7773 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
7774 Component) {
7775 if (MC.getAssociatedDeclaration()) {
7776 ComponentLB =
7777 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
7778 .getAddress();
7779 Size = CGF.Builder.CreatePtrDiff(
7780 CGF.EmitCastToVoidPtr(ComponentLB.getPointer()),
7781 CGF.EmitCastToVoidPtr(LB.getPointer()));
7782 break;
7783 }
7784 }
7785 BasePointers.push_back(BP.getPointer());
7786 Pointers.push_back(LB.getPointer());
7787 Sizes.push_back(CGF.Builder.CreateIntCast(Size, CGF.Int64Ty,
7788 /*isSigned=*/true));
7789 Types.push_back(Flags);
7790 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7791 }
7792 BasePointers.push_back(BP.getPointer());
7793 Pointers.push_back(LB.getPointer());
7794 Size = CGF.Builder.CreatePtrDiff(
7795 CGF.EmitCastToVoidPtr(
7796 CGF.Builder.CreateConstGEP(HB, 1).getPointer()),
7797 CGF.EmitCastToVoidPtr(LB.getPointer()));
7798 Sizes.push_back(
7799 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7800 Types.push_back(Flags);
7801 break;
7802 }
7803 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7804 if (!IsMemberPointer) {
7805 BasePointers.push_back(BP.getPointer());
7806 Pointers.push_back(LB.getPointer());
7807 Sizes.push_back(
7808 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7809
7810 // We need to add a pointer flag for each map that comes from the
7811 // same expression except for the first one. We also need to signal
7812 // this map is the first one that relates with the current capture
7813 // (there is a set of entries for each capture).
7814 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7815 MapType, MapModifiers, IsImplicit,
7816 !IsExpressionFirstInfo || RequiresReference,
7817 IsCaptureFirstInfo && !RequiresReference);
7818
7819 if (!IsExpressionFirstInfo) {
7820 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
7821 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
7822 if (IsPointer)
7823 Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7824 OMP_MAP_DELETE | OMP_MAP_CLOSE);
7825
7826 if (ShouldBeMemberOf) {
7827 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
7828 // should be later updated with the correct value of MEMBER_OF.
7829 Flags |= OMP_MAP_MEMBER_OF;
7830 // From now on, all subsequent PTR_AND_OBJ entries should not be
7831 // marked as MEMBER_OF.
7832 ShouldBeMemberOf = false;
7833 }
7834 }
7835
7836 Types.push_back(Flags);
7837 }
7838
7839 // If we have encountered a member expression so far, keep track of the
7840 // mapped member. If the parent is "*this", then the value declaration
7841 // is nullptr.
7842 if (EncounteredME) {
7843 const auto *FD = dyn_cast<FieldDecl>(EncounteredME->getMemberDecl());
7844 unsigned FieldIndex = FD->getFieldIndex();
7845
7846 // Update info about the lowest and highest elements for this struct
7847 if (!PartialStruct.Base.isValid()) {
7848 PartialStruct.LowestElem = {FieldIndex, LB};
7849 PartialStruct.HighestElem = {FieldIndex, LB};
7850 PartialStruct.Base = BP;
7851 } else if (FieldIndex < PartialStruct.LowestElem.first) {
7852 PartialStruct.LowestElem = {FieldIndex, LB};
7853 } else if (FieldIndex > PartialStruct.HighestElem.first) {
7854 PartialStruct.HighestElem = {FieldIndex, LB};
7855 }
7856 }
7857
7858 // If we have a final array section, we are done with this expression.
7859 if (IsFinalArraySection)
7860 break;
7861
7862 // The pointer becomes the base for the next element.
7863 if (Next != CE)
7864 BP = LB;
7865
7866 IsExpressionFirstInfo = false;
7867 IsCaptureFirstInfo = false;
7868 }
7869 }
7870 }
7871
7872 /// Return the adjusted map modifiers if the declaration a capture refers to
7873 /// appears in a first-private clause. This is expected to be used only with
7874 /// directives that start with 'target'.
7875 MappableExprsHandler::OpenMPOffloadMappingFlags
7876 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
7877 assert(Cap.capturesVariable() && "Expected capture by reference only!")((Cap.capturesVariable() && "Expected capture by reference only!"
) ? static_cast<void> (0) : __assert_fail ("Cap.capturesVariable() && \"Expected capture by reference only!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7877, __PRETTY_FUNCTION__))
;
7878
7879 // A first private variable captured by reference will use only the
7880 // 'private ptr' and 'map to' flag. Return the right flags if the captured
7881 // declaration is known as first-private in this handler.
7882 if (FirstPrivateDecls.count(Cap.getCapturedVar())) {
7883 if (Cap.getCapturedVar()->getType().isConstant(CGF.getContext()) &&
7884 Cap.getCaptureKind() == CapturedStmt::VCK_ByRef)
7885 return MappableExprsHandler::OMP_MAP_ALWAYS |
7886 MappableExprsHandler::OMP_MAP_TO;
7887 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
7888 return MappableExprsHandler::OMP_MAP_TO |
7889 MappableExprsHandler::OMP_MAP_PTR_AND_OBJ;
7890 return MappableExprsHandler::OMP_MAP_PRIVATE |
7891 MappableExprsHandler::OMP_MAP_TO;
7892 }
7893 return MappableExprsHandler::OMP_MAP_TO |
7894 MappableExprsHandler::OMP_MAP_FROM;
7895 }
7896
7897 static OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position) {
7898 // Rotate by getFlagMemberOffset() bits.
7899 return static_cast<OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
7900 << getFlagMemberOffset());
7901 }
7902
7903 static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7904 OpenMPOffloadMappingFlags MemberOfFlag) {
7905 // If the entry is PTR_AND_OBJ but has not been marked with the special
7906 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
7907 // marked as MEMBER_OF.
7908 if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7909 ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7910 return;
7911
7912 // Reset the placeholder value to prepare the flag for the assignment of the
7913 // proper MEMBER_OF value.
7914 Flags &= ~OMP_MAP_MEMBER_OF;
7915 Flags |= MemberOfFlag;
7916 }
7917
7918 void getPlainLayout(const CXXRecordDecl *RD,
7919 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
7920 bool AsBase) const {
7921 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
7922
7923 llvm::StructType *St =
7924 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
7925
7926 unsigned NumElements = St->getNumElements();
7927 llvm::SmallVector<
7928 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7929 RecordLayout(NumElements);
7930
7931 // Fill bases.
7932 for (const auto &I : RD->bases()) {
7933 if (I.isVirtual())
7934 continue;
7935 const auto *Base = I.getType()->getAsCXXRecordDecl();
7936 // Ignore empty bases.
7937 if (Base->isEmpty() || CGF.getContext()
7938 .getASTRecordLayout(Base)
7939 .getNonVirtualSize()
7940 .isZero())
7941 continue;
7942
7943 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base);
7944 RecordLayout[FieldIndex] = Base;
7945 }
7946 // Fill in virtual bases.
7947 for (const auto &I : RD->vbases()) {
7948 const auto *Base = I.getType()->getAsCXXRecordDecl();
7949 // Ignore empty bases.
7950 if (Base->isEmpty())
7951 continue;
7952 unsigned FieldIndex = RL.getVirtualBaseIndex(Base);
7953 if (RecordLayout[FieldIndex])
7954 continue;
7955 RecordLayout[FieldIndex] = Base;
7956 }
7957 // Fill in all the fields.
7958 assert(!RD->isUnion() && "Unexpected union.")((!RD->isUnion() && "Unexpected union.") ? static_cast
<void> (0) : __assert_fail ("!RD->isUnion() && \"Unexpected union.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 7958, __PRETTY_FUNCTION__))
;
7959 for (const auto *Field : RD->fields()) {
7960 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
7961 // will fill in later.)
7962 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) {
7963 unsigned FieldIndex = RL.getLLVMFieldNo(Field);
7964 RecordLayout[FieldIndex] = Field;
7965 }
7966 }
7967 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7968 &Data : RecordLayout) {
7969 if (Data.isNull())
7970 continue;
7971 if (const auto *Base = Data.dyn_cast<const CXXRecordDecl *>())
7972 getPlainLayout(Base, Layout, /*AsBase=*/true);
7973 else
7974 Layout.push_back(Data.get<const FieldDecl *>());
7975 }
7976 }
7977
7978public:
7979 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
7980 : CurDir(&Dir), CGF(CGF) {
7981 // Extract firstprivate clause information.
7982 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
7983 for (const auto *D : C->varlists())
7984 FirstPrivateDecls.try_emplace(
7985 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit());
7986 // Extract device pointer clause information.
7987 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
7988 for (auto L : C->component_lists())
7989 DevPointersMap[L.first].push_back(L.second);
7990 }
7991
7992 /// Constructor for the declare mapper directive.
7993 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
7994 : CurDir(&Dir), CGF(CGF) {}
7995
7996 /// Generate code for the combined entry if we have a partially mapped struct
7997 /// and take care of the mapping flags of the arguments corresponding to
7998 /// individual struct members.
7999 void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
8000 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8001 MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
8002 const StructRangeInfoTy &PartialStruct) const {
8003 // Base is the base of the struct
8004 BasePointers.push_back(PartialStruct.Base.getPointer());
8005 // Pointer is the address of the lowest element
8006 llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
8007 Pointers.push_back(LB);
8008 // Size is (addr of {highest+1} element) - (addr of lowest element)
8009 llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
8010 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1);
8011 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
8012 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
8013 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr);
8014 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,
8015 /*isSigned=*/false);
8016 Sizes.push_back(Size);
8017 // Map type is always TARGET_PARAM
8018 Types.push_back(OMP_MAP_TARGET_PARAM);
8019 // Remove TARGET_PARAM flag from the first element
8020 (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
8021
8022 // All other current entries will be MEMBER_OF the combined entry
8023 // (except for PTR_AND_OBJ entries which do not have a placeholder value
8024 // 0xFFFF in the MEMBER_OF field).
8025 OpenMPOffloadMappingFlags MemberOfFlag =
8026 getMemberOfFlag(BasePointers.size() - 1);
8027 for (auto &M : CurTypes)
8028 setCorrectMemberOfFlag(M, MemberOfFlag);
8029 }
8030
8031 /// Generate all the base pointers, section pointers, sizes and map
8032 /// types for the extracted mappable expressions. Also, for each item that
8033 /// relates with a device pointer, a pair of the relevant declaration and
8034 /// index where it occurs is appended to the device pointers info array.
8035 void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
8036 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8037 MapFlagsArrayTy &Types) const {
8038 // We have to process the component lists that relate with the same
8039 // declaration in a single chunk so that we can generate the map flags
8040 // correctly. Therefore, we organize all lists in a map.
8041 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
8042
8043 // Helper function to fill the information map for the different supported
8044 // clauses.
8045 auto &&InfoGen = [&Info](
8046 const ValueDecl *D,
8047 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8048 OpenMPMapClauseKind MapType,
8049 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8050 bool ReturnDevicePointer, bool IsImplicit) {
8051 const ValueDecl *VD =
8052 D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
8053 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
8054 IsImplicit);
8055 };
8056
8057 assert(CurDir.is<const OMPExecutableDirective *>() &&((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8058, __PRETTY_FUNCTION__))
8058 "Expect a executable directive")((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8058, __PRETTY_FUNCTION__))
;
8059 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
8060 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>())
8061 for (const auto &L : C->component_lists()) {
8062 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(),
8063 /*ReturnDevicePointer=*/false, C->isImplicit());
8064 }
8065 for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>())
8066 for (const auto &L : C->component_lists()) {
8067 InfoGen(L.first, L.second, OMPC_MAP_to, llvm::None,
8068 /*ReturnDevicePointer=*/false, C->isImplicit());
8069 }
8070 for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>())
8071 for (const auto &L : C->component_lists()) {
8072 InfoGen(L.first, L.second, OMPC_MAP_from, llvm::None,
8073 /*ReturnDevicePointer=*/false, C->isImplicit());
8074 }
8075
8076 // Look at the use_device_ptr clause information and mark the existing map
8077 // entries as such. If there is no map information for an entry in the
8078 // use_device_ptr list, we create one with map type 'alloc' and zero size
8079 // section. It is the user fault if that was not mapped before. If there is
8080 // no map information and the pointer is a struct member, then we defer the
8081 // emission of that entry until the whole struct has been processed.
8082 llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
8083 DeferredInfo;
8084
8085 for (const auto *C :
8086 CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) {
8087 for (const auto &L : C->component_lists()) {
8088 assert(!L.second.empty() && "Not expecting empty list of components!")((!L.second.empty() && "Not expecting empty list of components!"
) ? static_cast<void> (0) : __assert_fail ("!L.second.empty() && \"Not expecting empty list of components!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8088, __PRETTY_FUNCTION__))
;
8089 const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
8090 VD = cast<ValueDecl>(VD->getCanonicalDecl());
8091 const Expr *IE = L.second.back().getAssociatedExpression();
8092 // If the first component is a member expression, we have to look into
8093 // 'this', which maps to null in the map of map information. Otherwise
8094 // look directly for the information.
8095 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
8096
8097 // We potentially have map information for this declaration already.
8098 // Look for the first set of components that refer to it.
8099 if (It != Info.end()) {
8100 auto CI = std::find_if(
8101 It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
8102 return MI.Components.back().getAssociatedDeclaration() == VD;
8103 });
8104 // If we found a map entry, signal that the pointer has to be returned
8105 // and move on to the next declaration.
8106 if (CI != It->second.end()) {
8107 CI->ReturnDevicePointer = true;
8108 continue;
8109 }
8110 }
8111
8112 // We didn't find any match in our map information - generate a zero
8113 // size array section - if the pointer is a struct member we defer this
8114 // action until the whole struct has been processed.
8115 if (isa<MemberExpr>(IE)) {
8116 // Insert the pointer into Info to be processed by
8117 // generateInfoForComponentList. Because it is a member pointer
8118 // without a pointee, no entry will be generated for it, therefore
8119 // we need to generate one after the whole struct has been processed.
8120 // Nonetheless, generateInfoForComponentList must be called to take
8121 // the pointer into account for the calculation of the range of the
8122 // partial struct.
8123 InfoGen(nullptr, L.second, OMPC_MAP_unknown, llvm::None,
8124 /*ReturnDevicePointer=*/false, C->isImplicit());
8125 DeferredInfo[nullptr].emplace_back(IE, VD);
8126 } else {
8127 llvm::Value *Ptr =
8128 CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
8129 BasePointers.emplace_back(Ptr, VD);
8130 Pointers.push_back(Ptr);
8131 Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8132 Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
8133 }
8134 }
8135 }
8136
8137 for (const auto &M : Info) {
8138 // We need to know when we generate information for the first component
8139 // associated with a capture, because the mapping flags depend on it.
8140 bool IsFirstComponentList = true;
8141
8142 // Temporary versions of arrays
8143 MapBaseValuesArrayTy CurBasePointers;
8144 MapValuesArrayTy CurPointers;
8145 MapValuesArrayTy CurSizes;
8146 MapFlagsArrayTy CurTypes;
8147 StructRangeInfoTy PartialStruct;
8148
8149 for (const MapInfo &L : M.second) {
8150 assert(!L.Components.empty() &&((!L.Components.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8151, __PRETTY_FUNCTION__))
8151 "Not expecting declaration with no component lists.")((!L.Components.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8151, __PRETTY_FUNCTION__))
;
8152
8153 // Remember the current base pointer index.
8154 unsigned CurrentBasePointersIdx = CurBasePointers.size();
8155 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8156 CurBasePointers, CurPointers, CurSizes,
8157 CurTypes, PartialStruct,
8158 IsFirstComponentList, L.IsImplicit);
8159
8160 // If this entry relates with a device pointer, set the relevant
8161 // declaration and add the 'return pointer' flag.
8162 if (L.ReturnDevicePointer) {
8163 assert(CurBasePointers.size() > CurrentBasePointersIdx &&((CurBasePointers.size() > CurrentBasePointersIdx &&
"Unexpected number of mapped base pointers.") ? static_cast<
void> (0) : __assert_fail ("CurBasePointers.size() > CurrentBasePointersIdx && \"Unexpected number of mapped base pointers.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8164, __PRETTY_FUNCTION__))
8164 "Unexpected number of mapped base pointers.")((CurBasePointers.size() > CurrentBasePointersIdx &&
"Unexpected number of mapped base pointers.") ? static_cast<
void> (0) : __assert_fail ("CurBasePointers.size() > CurrentBasePointersIdx && \"Unexpected number of mapped base pointers.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8164, __PRETTY_FUNCTION__))
;
8165
8166 const ValueDecl *RelevantVD =
8167 L.Components.back().getAssociatedDeclaration();
8168 assert(RelevantVD &&((RelevantVD && "No relevant declaration related with device pointer??"
) ? static_cast<void> (0) : __assert_fail ("RelevantVD && \"No relevant declaration related with device pointer??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8169, __PRETTY_FUNCTION__))
8169 "No relevant declaration related with device pointer??")((RelevantVD && "No relevant declaration related with device pointer??"
) ? static_cast<void> (0) : __assert_fail ("RelevantVD && \"No relevant declaration related with device pointer??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8169, __PRETTY_FUNCTION__))
;
8170
8171 CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
8172 CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
8173 }
8174 IsFirstComponentList = false;
8175 }
8176
8177 // Append any pending zero-length pointers which are struct members and
8178 // used with use_device_ptr.
8179 auto CI = DeferredInfo.find(M.first);
8180 if (CI != DeferredInfo.end()) {
8181 for (const DeferredDevicePtrEntryTy &L : CI->second) {
8182 llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer();
8183 llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
8184 this->CGF.EmitLValue(L.IE), L.IE->getExprLoc());
8185 CurBasePointers.emplace_back(BasePtr, L.VD);
8186 CurPointers.push_back(Ptr);
8187 CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.Int64Ty));
8188 // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the placeholder
8189 // value MEMBER_OF=FFFF so that the entry is later updated with the
8190 // correct value of MEMBER_OF.
8191 CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
8192 OMP_MAP_MEMBER_OF);
8193 }
8194 }
8195
8196 // If there is an entry in PartialStruct it means we have a struct with
8197 // individual members mapped. Emit an extra combined entry.
8198 if (PartialStruct.Base.isValid())
8199 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8200 PartialStruct);
8201
8202 // We need to append the results of this capture to what we already have.
8203 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8204 Pointers.append(CurPointers.begin(), CurPointers.end());
8205 Sizes.append(CurSizes.begin(), CurSizes.end());
8206 Types.append(CurTypes.begin(), CurTypes.end());
8207 }
8208 }
8209
8210 /// Generate all the base pointers, section pointers, sizes and map types for
8211 /// the extracted map clauses of user-defined mapper.
8212 void generateAllInfoForMapper(MapBaseValuesArrayTy &BasePointers,
8213 MapValuesArrayTy &Pointers,
8214 MapValuesArrayTy &Sizes,
8215 MapFlagsArrayTy &Types) const {
8216 assert(CurDir.is<const OMPDeclareMapperDecl *>() &&((CurDir.is<const OMPDeclareMapperDecl *>() && "Expect a declare mapper directive"
) ? static_cast<void> (0) : __assert_fail ("CurDir.is<const OMPDeclareMapperDecl *>() && \"Expect a declare mapper directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8217, __PRETTY_FUNCTION__))
8217 "Expect a declare mapper directive")((CurDir.is<const OMPDeclareMapperDecl *>() && "Expect a declare mapper directive"
) ? static_cast<void> (0) : __assert_fail ("CurDir.is<const OMPDeclareMapperDecl *>() && \"Expect a declare mapper directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8217, __PRETTY_FUNCTION__))
;
8218 const auto *CurMapperDir = CurDir.get<const OMPDeclareMapperDecl *>();
8219 // We have to process the component lists that relate with the same
8220 // declaration in a single chunk so that we can generate the map flags
8221 // correctly. Therefore, we organize all lists in a map.
8222 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
8223
8224 // Helper function to fill the information map for the different supported
8225 // clauses.
8226 auto &&InfoGen = [&Info](
8227 const ValueDecl *D,
8228 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8229 OpenMPMapClauseKind MapType,
8230 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8231 bool ReturnDevicePointer, bool IsImplicit) {
8232 const ValueDecl *VD =
8233 D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
8234 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
8235 IsImplicit);
8236 };
8237
8238 for (const auto *C : CurMapperDir->clauselists()) {
8239 const auto *MC = cast<OMPMapClause>(C);
8240 for (const auto &L : MC->component_lists()) {
8241 InfoGen(L.first, L.second, MC->getMapType(), MC->getMapTypeModifiers(),
8242 /*ReturnDevicePointer=*/false, MC->isImplicit());
8243 }
8244 }
8245
8246 for (const auto &M : Info) {
8247 // We need to know when we generate information for the first component
8248 // associated with a capture, because the mapping flags depend on it.
8249 bool IsFirstComponentList = true;
8250
8251 // Temporary versions of arrays
8252 MapBaseValuesArrayTy CurBasePointers;
8253 MapValuesArrayTy CurPointers;
8254 MapValuesArrayTy CurSizes;
8255 MapFlagsArrayTy CurTypes;
8256 StructRangeInfoTy PartialStruct;
8257
8258 for (const MapInfo &L : M.second) {
8259 assert(!L.Components.empty() &&((!L.Components.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8260, __PRETTY_FUNCTION__))
8260 "Not expecting declaration with no component lists.")((!L.Components.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8260, __PRETTY_FUNCTION__))
;
8261 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8262 CurBasePointers, CurPointers, CurSizes,
8263 CurTypes, PartialStruct,
8264 IsFirstComponentList, L.IsImplicit);
8265 IsFirstComponentList = false;
8266 }
8267
8268 // If there is an entry in PartialStruct it means we have a struct with
8269 // individual members mapped. Emit an extra combined entry.
8270 if (PartialStruct.Base.isValid())
8271 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8272 PartialStruct);
8273
8274 // We need to append the results of this capture to what we already have.
8275 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8276 Pointers.append(CurPointers.begin(), CurPointers.end());
8277 Sizes.append(CurSizes.begin(), CurSizes.end());
8278 Types.append(CurTypes.begin(), CurTypes.end());
8279 }
8280 }
8281
8282 /// Emit capture info for lambdas for variables captured by reference.
8283 void generateInfoForLambdaCaptures(
8284 const ValueDecl *VD, llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers,
8285 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8286 MapFlagsArrayTy &Types,
8287 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
8288 const auto *RD = VD->getType()
8289 .getCanonicalType()
8290 .getNonReferenceType()
8291 ->getAsCXXRecordDecl();
8292 if (!RD || !RD->isLambda())
8293 return;
8294 Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD));
8295 LValue VDLVal = CGF.MakeAddrLValue(
8296 VDAddr, VD->getType().getCanonicalType().getNonReferenceType());
8297 llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
8298 FieldDecl *ThisCapture = nullptr;
8299 RD->getCaptureFields(Captures, ThisCapture);
8300 if (ThisCapture) {
8301 LValue ThisLVal =
8302 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
8303 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
8304 LambdaPointers.try_emplace(ThisLVal.getPointer(), VDLVal.getPointer());
8305 BasePointers.push_back(ThisLVal.getPointer());
8306 Pointers.push_back(ThisLValVal.getPointer());
8307 Sizes.push_back(
8308 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8309 CGF.Int64Ty, /*isSigned=*/true));
8310 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8311 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8312 }
8313 for (const LambdaCapture &LC : RD->captures()) {
8314 if (!LC.capturesVariable())
8315 continue;
8316 const VarDecl *VD = LC.getCapturedVar();
8317 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
8318 continue;
8319 auto It = Captures.find(VD);
8320 assert(It != Captures.end() && "Found lambda capture without field.")((It != Captures.end() && "Found lambda capture without field."
) ? static_cast<void> (0) : __assert_fail ("It != Captures.end() && \"Found lambda capture without field.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8320, __PRETTY_FUNCTION__))
;
8321 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
8322 if (LC.getCaptureKind() == LCK_ByRef) {
8323 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
8324 LambdaPointers.try_emplace(VarLVal.getPointer(), VDLVal.getPointer());
8325 BasePointers.push_back(VarLVal.getPointer());
8326 Pointers.push_back(VarLValVal.getPointer());
8327 Sizes.push_back(CGF.Builder.CreateIntCast(
8328 CGF.getTypeSize(
8329 VD->getType().getCanonicalType().getNonReferenceType()),
8330 CGF.Int64Ty, /*isSigned=*/true));
8331 } else {
8332 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());
8333 LambdaPointers.try_emplace(VarLVal.getPointer(), VDLVal.getPointer());
8334 BasePointers.push_back(VarLVal.getPointer());
8335 Pointers.push_back(VarRVal.getScalarVal());
8336 Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
8337 }
8338 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8339 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8340 }
8341 }
8342
8343 /// Set correct indices for lambdas captures.
8344 void adjustMemberOfForLambdaCaptures(
8345 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8346 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8347 MapFlagsArrayTy &Types) const {
8348 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
8349 // Set correct member_of idx for all implicit lambda captures.
8350 if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8351 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT))
8352 continue;
8353 llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
8354 assert(BasePtr && "Unable to find base lambda address.")((BasePtr && "Unable to find base lambda address.") ?
static_cast<void> (0) : __assert_fail ("BasePtr && \"Unable to find base lambda address.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8354, __PRETTY_FUNCTION__))
;
8355 int TgtIdx = -1;
8356 for (unsigned J = I; J > 0; --J) {
8357 unsigned Idx = J - 1;
8358 if (Pointers[Idx] != BasePtr)
8359 continue;
8360 TgtIdx = Idx;
8361 break;
8362 }
8363 assert(TgtIdx != -1 && "Unable to find parent lambda.")((TgtIdx != -1 && "Unable to find parent lambda.") ? static_cast
<void> (0) : __assert_fail ("TgtIdx != -1 && \"Unable to find parent lambda.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8363, __PRETTY_FUNCTION__))
;
8364 // All other current entries will be MEMBER_OF the combined entry
8365 // (except for PTR_AND_OBJ entries which do not have a placeholder value
8366 // 0xFFFF in the MEMBER_OF field).
8367 OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx);
8368 setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8369 }
8370 }
8371
8372 /// Generate the base pointers, section pointers, sizes and map types
8373 /// associated to a given capture.
8374 void generateInfoForCapture(const CapturedStmt::Capture *Cap,
8375 llvm::Value *Arg,
8376 MapBaseValuesArrayTy &BasePointers,
8377 MapValuesArrayTy &Pointers,
8378 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
8379 StructRangeInfoTy &PartialStruct) const {
8380 assert(!Cap->capturesVariableArrayType() &&((!Cap->capturesVariableArrayType() && "Not expecting to generate map info for a variable array type!"
) ? static_cast<void> (0) : __assert_fail ("!Cap->capturesVariableArrayType() && \"Not expecting to generate map info for a variable array type!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8381, __PRETTY_FUNCTION__))
8381 "Not expecting to generate map info for a variable array type!")((!Cap->capturesVariableArrayType() && "Not expecting to generate map info for a variable array type!"
) ? static_cast<void> (0) : __assert_fail ("!Cap->capturesVariableArrayType() && \"Not expecting to generate map info for a variable array type!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8381, __PRETTY_FUNCTION__))
;
8382
8383 // We need to know when we generating information for the first component
8384 const ValueDecl *VD = Cap->capturesThis()
8385 ? nullptr
8386 : Cap->getCapturedVar()->getCanonicalDecl();
8387
8388 // If this declaration appears in a is_device_ptr clause we just have to
8389 // pass the pointer by value. If it is a reference to a declaration, we just
8390 // pass its value.
8391 if (DevPointersMap.count(VD)) {
8392 BasePointers.emplace_back(Arg, VD);
8393 Pointers.push_back(Arg);
8394 Sizes.push_back(
8395 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8396 CGF.Int64Ty, /*isSigned=*/true));
8397 Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
8398 return;
8399 }
8400
8401 using MapData =
8402 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
8403 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, bool>;
8404 SmallVector<MapData, 4> DeclComponentLists;
8405 assert(CurDir.is<const OMPExecutableDirective *>() &&((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8406, __PRETTY_FUNCTION__))
8406 "Expect a executable directive")((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8406, __PRETTY_FUNCTION__))
;
8407 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
8408 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
8409 for (const auto &L : C->decl_component_lists(VD)) {
8410 assert(L.first == VD &&((L.first == VD && "We got information for the wrong declaration??"
) ? static_cast<void> (0) : __assert_fail ("L.first == VD && \"We got information for the wrong declaration??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8411, __PRETTY_FUNCTION__))
8411 "We got information for the wrong declaration??")((L.first == VD && "We got information for the wrong declaration??"
) ? static_cast<void> (0) : __assert_fail ("L.first == VD && \"We got information for the wrong declaration??\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8411, __PRETTY_FUNCTION__))
;
8412 assert(!L.second.empty() &&((!L.second.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.second.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8413, __PRETTY_FUNCTION__))
8413 "Not expecting declaration with no component lists.")((!L.second.empty() && "Not expecting declaration with no component lists."
) ? static_cast<void> (0) : __assert_fail ("!L.second.empty() && \"Not expecting declaration with no component lists.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8413, __PRETTY_FUNCTION__))
;
8414 DeclComponentLists.emplace_back(L.second, C->getMapType(),
8415 C->getMapTypeModifiers(),
8416 C->isImplicit());
8417 }
8418 }
8419
8420 // Find overlapping elements (including the offset from the base element).
8421 llvm::SmallDenseMap<
8422 const MapData *,
8423 llvm::SmallVector<
8424 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
8425 4>
8426 OverlappedData;
8427 size_t Count = 0;
8428 for (const MapData &L : DeclComponentLists) {
8429 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8430 OpenMPMapClauseKind MapType;
8431 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8432 bool IsImplicit;
8433 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8434 ++Count;
8435 for (const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) {
8436 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
8437 std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1;
8438 auto CI = Components.rbegin();
8439 auto CE = Components.rend();
8440 auto SI = Components1.rbegin();
8441 auto SE = Components1.rend();
8442 for (; CI != CE && SI != SE; ++CI, ++SI) {
8443 if (CI->getAssociatedExpression()->getStmtClass() !=
8444 SI->getAssociatedExpression()->getStmtClass())
8445 break;
8446 // Are we dealing with different variables/fields?
8447 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8448 break;
8449 }
8450 // Found overlapping if, at least for one component, reached the head of
8451 // the components list.
8452 if (CI == CE || SI == SE) {
8453 assert((CI != CE || SI != SE) &&(((CI != CE || SI != SE) && "Unexpected full match of the mapping components."
) ? static_cast<void> (0) : __assert_fail ("(CI != CE || SI != SE) && \"Unexpected full match of the mapping components.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8454, __PRETTY_FUNCTION__))
8454 "Unexpected full match of the mapping components.")(((CI != CE || SI != SE) && "Unexpected full match of the mapping components."
) ? static_cast<void> (0) : __assert_fail ("(CI != CE || SI != SE) && \"Unexpected full match of the mapping components.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8454, __PRETTY_FUNCTION__))
;
8455 const MapData &BaseData = CI == CE ? L : L1;
8456 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
8457 SI == SE ? Components : Components1;
8458 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8459 OverlappedElements.getSecond().push_back(SubData);
8460 }
8461 }
8462 }
8463 // Sort the overlapped elements for each item.
8464 llvm::SmallVector<const FieldDecl *, 4> Layout;
8465 if (!OverlappedData.empty()) {
8466 if (const auto *CRD =
8467 VD->getType().getCanonicalType()->getAsCXXRecordDecl())
8468 getPlainLayout(CRD, Layout, /*AsBase=*/false);
8469 else {
8470 const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl();
8471 Layout.append(RD->field_begin(), RD->field_end());
8472 }
8473 }
8474 for (auto &Pair : OverlappedData) {
8475 llvm::sort(
8476 Pair.getSecond(),
8477 [&Layout](
8478 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
8479 OMPClauseMappableExprCommon::MappableExprComponentListRef
8480 Second) {
8481 auto CI = First.rbegin();
8482 auto CE = First.rend();
8483 auto SI = Second.rbegin();
8484 auto SE = Second.rend();
8485 for (; CI != CE && SI != SE; ++CI, ++SI) {
8486 if (CI->getAssociatedExpression()->getStmtClass() !=
8487 SI->getAssociatedExpression()->getStmtClass())
8488 break;
8489 // Are we dealing with different variables/fields?
8490 if (CI->getAssociatedDeclaration() !=
8491 SI->getAssociatedDeclaration())
8492 break;
8493 }
8494
8495 // Lists contain the same elements.
8496 if (CI == CE && SI == SE)
8497 return false;
8498
8499 // List with less elements is less than list with more elements.
8500 if (CI == CE || SI == SE)
8501 return CI == CE;
8502
8503 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8504 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8505 if (FD1->getParent() == FD2->getParent())
8506 return FD1->getFieldIndex() < FD2->getFieldIndex();
8507 const auto It =
8508 llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) {
8509 return FD == FD1 || FD == FD2;
8510 });
8511 return *It == FD1;
8512 });
8513 }
8514
8515 // Associated with a capture, because the mapping flags depend on it.
8516 // Go through all of the elements with the overlapped elements.
8517 for (const auto &Pair : OverlappedData) {
8518 const MapData &L = *Pair.getFirst();
8519 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8520 OpenMPMapClauseKind MapType;
8521 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8522 bool IsImplicit;
8523 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8524 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
8525 OverlappedComponents = Pair.getSecond();
8526 bool IsFirstComponentList = true;
8527 generateInfoForComponentList(MapType, MapModifiers, Components,
8528 BasePointers, Pointers, Sizes, Types,
8529 PartialStruct, IsFirstComponentList,
8530 IsImplicit, OverlappedComponents);
8531 }
8532 // Go through other elements without overlapped elements.
8533 bool IsFirstComponentList = OverlappedData.empty();
8534 for (const MapData &L : DeclComponentLists) {
8535 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8536 OpenMPMapClauseKind MapType;
8537 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8538 bool IsImplicit;
8539 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8540 auto It = OverlappedData.find(&L);
8541 if (It == OverlappedData.end())
8542 generateInfoForComponentList(MapType, MapModifiers, Components,
8543 BasePointers, Pointers, Sizes, Types,
8544 PartialStruct, IsFirstComponentList,
8545 IsImplicit);
8546 IsFirstComponentList = false;
8547 }
8548 }
8549
8550 /// Generate the base pointers, section pointers, sizes and map types
8551 /// associated with the declare target link variables.
8552 void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
8553 MapValuesArrayTy &Pointers,
8554 MapValuesArrayTy &Sizes,
8555 MapFlagsArrayTy &Types) const {
8556 assert(CurDir.is<const OMPExecutableDirective *>() &&((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8557, __PRETTY_FUNCTION__))
8557 "Expect a executable directive")((CurDir.is<const OMPExecutableDirective *>() &&
"Expect a executable directive") ? static_cast<void> (
0) : __assert_fail ("CurDir.is<const OMPExecutableDirective *>() && \"Expect a executable directive\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8557, __PRETTY_FUNCTION__))
;
8558 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
8559 // Map other list items in the map clause which are not captured variables
8560 // but "declare target link" global variables.
8561 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
8562 for (const auto &L : C->component_lists()) {
8563 if (!L.first)
8564 continue;
8565 const auto *VD = dyn_cast<VarDecl>(L.first);
8566 if (!VD)
8567 continue;
8568 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8569 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8570 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8571 !Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
8572 continue;
8573 StructRangeInfoTy PartialStruct;
8574 generateInfoForComponentList(
8575 C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers,
8576 Pointers, Sizes, Types, PartialStruct,
8577 /*IsFirstComponentList=*/true, C->isImplicit());
8578 assert(!PartialStruct.Base.isValid() &&((!PartialStruct.Base.isValid() && "No partial structs for declare target link expected."
) ? static_cast<void> (0) : __assert_fail ("!PartialStruct.Base.isValid() && \"No partial structs for declare target link expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8579, __PRETTY_FUNCTION__))
8579 "No partial structs for declare target link expected.")((!PartialStruct.Base.isValid() && "No partial structs for declare target link expected."
) ? static_cast<void> (0) : __assert_fail ("!PartialStruct.Base.isValid() && \"No partial structs for declare target link expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8579, __PRETTY_FUNCTION__))
;
8580 }
8581 }
8582 }
8583
8584 /// Generate the default map information for a given capture \a CI,
8585 /// record field declaration \a RI and captured value \a CV.
8586 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
8587 const FieldDecl &RI, llvm::Value *CV,
8588 MapBaseValuesArrayTy &CurBasePointers,
8589 MapValuesArrayTy &CurPointers,
8590 MapValuesArrayTy &CurSizes,
8591 MapFlagsArrayTy &CurMapTypes) const {
8592 bool IsImplicit = true;
8593 // Do the default mapping.
8594 if (CI.capturesThis()) {
8595 CurBasePointers.push_back(CV);
8596 CurPointers.push_back(CV);
8597 const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
8598 CurSizes.push_back(
8599 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),
8600 CGF.Int64Ty, /*isSigned=*/true));
8601 // Default map type.
8602 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
8603 } else if (CI.capturesVariableByCopy()) {
8604 CurBasePointers.push_back(CV);
8605 CurPointers.push_back(CV);
8606 if (!RI.getType()->isAnyPointerType()) {
8607 // We have to signal to the runtime captures passed by value that are
8608 // not pointers.
8609 CurMapTypes.push_back(OMP_MAP_LITERAL);
8610 CurSizes.push_back(CGF.Builder.CreateIntCast(
8611 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));
8612 } else {
8613 // Pointers are implicitly mapped with a zero size and no flags
8614 // (other than first map that is added for all implicit maps).
8615 CurMapTypes.push_back(OMP_MAP_NONE);
8616 CurSizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8617 }
8618 const VarDecl *VD = CI.getCapturedVar();
8619 auto I = FirstPrivateDecls.find(VD);
8620 if (I != FirstPrivateDecls.end())
8621 IsImplicit = I->getSecond();
8622 } else {
8623 assert(CI.capturesVariable() && "Expected captured reference.")((CI.capturesVariable() && "Expected captured reference."
) ? static_cast<void> (0) : __assert_fail ("CI.capturesVariable() && \"Expected captured reference.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8623, __PRETTY_FUNCTION__))
;
8624 const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
8625 QualType ElementType = PtrTy->getPointeeType();
8626 CurSizes.push_back(CGF.Builder.CreateIntCast(
8627 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));
8628 // The default map type for a scalar/complex type is 'to' because by
8629 // default the value doesn't have to be retrieved. For an aggregate
8630 // type, the default is 'tofrom'.
8631 CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
8632 const VarDecl *VD = CI.getCapturedVar();
8633 auto I = FirstPrivateDecls.find(VD);
8634 if (I != FirstPrivateDecls.end() &&
8635 VD->getType().isConstant(CGF.getContext())) {
8636 llvm::Constant *Addr =
8637 CGF.CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(CGF, VD);
8638 // Copy the value of the original variable to the new global copy.
8639 CGF.Builder.CreateMemCpy(
8640 CGF.MakeNaturalAlignAddrLValue(Addr, ElementType).getAddress(),
8641 Address(CV, CGF.getContext().getTypeAlignInChars(ElementType)),
8642 CurSizes.back(), /*IsVolatile=*/false);
8643 // Use new global variable as the base pointers.
8644 CurBasePointers.push_back(Addr);
8645 CurPointers.push_back(Addr);
8646 } else {
8647 CurBasePointers.push_back(CV);
8648 if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {
8649 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
8650 CV, ElementType, CGF.getContext().getDeclAlign(VD),
8651 AlignmentSource::Decl));
8652 CurPointers.push_back(PtrAddr.getPointer());
8653 } else {
8654 CurPointers.push_back(CV);
8655 }
8656 }
8657 if (I != FirstPrivateDecls.end())
8658 IsImplicit = I->getSecond();
8659 }
8660 // Every default map produces a single argument which is a target parameter.
8661 CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
8662
8663 // Add flag stating this is an implicit map.
8664 if (IsImplicit)
8665 CurMapTypes.back() |= OMP_MAP_IMPLICIT;
8666 }
8667};
8668} // anonymous namespace
8669
8670/// Emit the arrays used to pass the captures and map information to the
8671/// offloading runtime library. If there is no map or capture information,
8672/// return nullptr by reference.
8673static void
8674emitOffloadingArrays(CodeGenFunction &CGF,
8675 MappableExprsHandler::MapBaseValuesArrayTy &BasePointers,
8676 MappableExprsHandler::MapValuesArrayTy &Pointers,
8677 MappableExprsHandler::MapValuesArrayTy &Sizes,
8678 MappableExprsHandler::MapFlagsArrayTy &MapTypes,
8679 CGOpenMPRuntime::TargetDataInfo &Info) {
8680 CodeGenModule &CGM = CGF.CGM;
8681 ASTContext &Ctx = CGF.getContext();
8682
8683 // Reset the array information.
8684 Info.clearArrayInfo();
8685 Info.NumberOfPtrs = BasePointers.size();
8686
8687 if (Info.NumberOfPtrs) {
8688 // Detect if we have any capture size requiring runtime evaluation of the
8689 // size so that a constant array could be eventually used.
8690 bool hasRuntimeEvaluationCaptureSize = false;
8691 for (llvm::Value *S : Sizes)
8692 if (!isa<llvm::Constant>(S)) {
8693 hasRuntimeEvaluationCaptureSize = true;
8694 break;
8695 }
8696
8697 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
8698 QualType PointerArrayType =
8699 Ctx.getConstantArrayType(Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal,
8700 /*IndexTypeQuals=*/0);
8701
8702 Info.BasePointersArray =
8703 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
8704 Info.PointersArray =
8705 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
8706
8707 // If we don't have any VLA types or other types that require runtime
8708 // evaluation, we can use a constant array for the map sizes, otherwise we
8709 // need to fill up the arrays as we do for the pointers.
8710 QualType Int64Ty =
8711 Ctx.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
8712 if (hasRuntimeEvaluationCaptureSize) {
8713 QualType SizeArrayType =
8714 Ctx.getConstantArrayType(Int64Ty, PointerNumAP, ArrayType::Normal,
8715 /*IndexTypeQuals=*/0);
8716 Info.SizesArray =
8717 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
8718 } else {
8719 // We expect all the sizes to be constant, so we collect them to create
8720 // a constant array.
8721 SmallVector<llvm::Constant *, 16> ConstSizes;
8722 for (llvm::Value *S : Sizes)
8723 ConstSizes.push_back(cast<llvm::Constant>(S));
8724
8725 auto *SizesArrayInit = llvm::ConstantArray::get(
8726 llvm::ArrayType::get(CGM.Int64Ty, ConstSizes.size()), ConstSizes);
8727 std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
8728 auto *SizesArrayGbl = new llvm::GlobalVariable(
8729 CGM.getModule(), SizesArrayInit->getType(),
8730 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8731 SizesArrayInit, Name);
8732 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8733 Info.SizesArray = SizesArrayGbl;
8734 }
8735
8736 // The map types are always constant so we don't need to generate code to
8737 // fill arrays. Instead, we create an array constant.
8738 SmallVector<uint64_t, 4> Mapping(MapTypes.size(), 0);
8739 llvm::copy(MapTypes, Mapping.begin());
8740 llvm::Constant *MapTypesArrayInit =
8741 llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping);
8742 std::string MaptypesName =
8743 CGM.getOpenMPRuntime().getName({"offload_maptypes"});
8744 auto *MapTypesArrayGbl = new llvm::GlobalVariable(
8745 CGM.getModule(), MapTypesArrayInit->getType(),
8746 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8747 MapTypesArrayInit, MaptypesName);
8748 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8749 Info.MapTypesArray = MapTypesArrayGbl;
8750
8751 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
8752 llvm::Value *BPVal = *BasePointers[I];
8753 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
8754 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8755 Info.BasePointersArray, 0, I);
8756 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8757 BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
8758 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8759 CGF.Builder.CreateStore(BPVal, BPAddr);
8760
8761 if (Info.requiresDevicePointerInfo())
8762 if (const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
8763 Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
8764
8765 llvm::Value *PVal = Pointers[I];
8766 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
8767 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8768 Info.PointersArray, 0, I);
8769 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8770 P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
8771 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8772 CGF.Builder.CreateStore(PVal, PAddr);
8773
8774 if (hasRuntimeEvaluationCaptureSize) {
8775 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
8776 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
8777 Info.SizesArray,
8778 /*Idx0=*/0,
8779 /*Idx1=*/I);
8780 Address SAddr(S, Ctx.getTypeAlignInChars(Int64Ty));
8781 CGF.Builder.CreateStore(
8782 CGF.Builder.CreateIntCast(Sizes[I], CGM.Int64Ty, /*isSigned=*/true),
8783 SAddr);
8784 }
8785 }
8786 }
8787}
8788
8789/// Emit the arguments to be passed to the runtime library based on the
8790/// arrays of pointers, sizes and map types.
8791static void emitOffloadingArraysArgument(
8792 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
8793 llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg,
8794 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
8795 CodeGenModule &CGM = CGF.CGM;
8796 if (Info.NumberOfPtrs) {
8797 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8798 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8799 Info.BasePointersArray,
8800 /*Idx0=*/0, /*Idx1=*/0);
8801 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8802 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8803 Info.PointersArray,
8804 /*Idx0=*/0,
8805 /*Idx1=*/0);
8806 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8807 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs), Info.SizesArray,
8808 /*Idx0=*/0, /*Idx1=*/0);
8809 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8810 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
8811 Info.MapTypesArray,
8812 /*Idx0=*/0,
8813 /*Idx1=*/0);
8814 } else {
8815 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8816 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8817 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
8818 MapTypesArrayArg =
8819 llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
8820 }
8821}
8822
8823/// Check for inner distribute directive.
8824static const OMPExecutableDirective *
8825getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
8826 const auto *CS = D.getInnermostCapturedStmt();
8827 const auto *Body =
8828 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
8829 const Stmt *ChildStmt =
8830 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
8831
8832 if (const auto *NestedDir =
8833 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8834 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
8835 switch (D.getDirectiveKind()) {
8836 case OMPD_target:
8837 if (isOpenMPDistributeDirective(DKind))
8838 return NestedDir;
8839 if (DKind == OMPD_teams) {
8840 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8841 /*IgnoreCaptured=*/true);
8842 if (!Body)
8843 return nullptr;
8844 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
8845 if (const auto *NND =
8846 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8847 DKind = NND->getDirectiveKind();
8848 if (isOpenMPDistributeDirective(DKind))
8849 return NND;
8850 }
8851 }
8852 return nullptr;
8853 case OMPD_target_teams:
8854 if (isOpenMPDistributeDirective(DKind))
8855 return NestedDir;
8856 return nullptr;
8857 case OMPD_target_parallel:
8858 case OMPD_target_simd:
8859 case OMPD_target_parallel_for:
8860 case OMPD_target_parallel_for_simd:
8861 return nullptr;
8862 case OMPD_target_teams_distribute:
8863 case OMPD_target_teams_distribute_simd:
8864 case OMPD_target_teams_distribute_parallel_for:
8865 case OMPD_target_teams_distribute_parallel_for_simd:
8866 case OMPD_parallel:
8867 case OMPD_for:
8868 case OMPD_parallel_for:
8869 case OMPD_parallel_sections:
8870 case OMPD_for_simd:
8871 case OMPD_parallel_for_simd:
8872 case OMPD_cancel:
8873 case OMPD_cancellation_point:
8874 case OMPD_ordered:
8875 case OMPD_threadprivate:
8876 case OMPD_allocate:
8877 case OMPD_task:
8878 case OMPD_simd:
8879 case OMPD_sections:
8880 case OMPD_section:
8881 case OMPD_single:
8882 case OMPD_master:
8883 case OMPD_critical:
8884 case OMPD_taskyield:
8885 case OMPD_barrier:
8886 case OMPD_taskwait:
8887 case OMPD_taskgroup:
8888 case OMPD_atomic:
8889 case OMPD_flush:
8890 case OMPD_teams:
8891 case OMPD_target_data:
8892 case OMPD_target_exit_data:
8893 case OMPD_target_enter_data:
8894 case OMPD_distribute:
8895 case OMPD_distribute_simd:
8896 case OMPD_distribute_parallel_for:
8897 case OMPD_distribute_parallel_for_simd:
8898 case OMPD_teams_distribute:
8899 case OMPD_teams_distribute_simd:
8900 case OMPD_teams_distribute_parallel_for:
8901 case OMPD_teams_distribute_parallel_for_simd:
8902 case OMPD_target_update:
8903 case OMPD_declare_simd:
8904 case OMPD_declare_variant:
8905 case OMPD_declare_target:
8906 case OMPD_end_declare_target:
8907 case OMPD_declare_reduction:
8908 case OMPD_declare_mapper:
8909 case OMPD_taskloop:
8910 case OMPD_taskloop_simd:
8911 case OMPD_requires:
8912 case OMPD_unknown:
8913 llvm_unreachable("Unexpected directive.")::llvm::llvm_unreachable_internal("Unexpected directive.", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 8913)
;
8914 }
8915 }
8916
8917 return nullptr;
8918}
8919
8920/// Emit the user-defined mapper function. The code generation follows the
8921/// pattern in the example below.
8922/// \code
8923/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
8924/// void *base, void *begin,
8925/// int64_t size, int64_t type) {
8926/// // Allocate space for an array section first.
8927/// if (size > 1 && !maptype.IsDelete)
8928/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
8929/// size*sizeof(Ty), clearToFrom(type));
8930/// // Map members.
8931/// for (unsigned i = 0; i < size; i++) {
8932/// // For each component specified by this mapper:
8933/// for (auto c : all_components) {
8934/// if (c.hasMapper())
8935/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
8936/// c.arg_type);
8937/// else
8938/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
8939/// c.arg_begin, c.arg_size, c.arg_type);
8940/// }
8941/// }
8942/// // Delete the array section.
8943/// if (size > 1 && maptype.IsDelete)
8944/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
8945/// size*sizeof(Ty), clearToFrom(type));
8946/// }
8947/// \endcode
8948void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
8949 CodeGenFunction *CGF) {
8950 if (UDMMap.count(D) > 0)
8951 return;
8952 ASTContext &C = CGM.getContext();
8953 QualType Ty = D->getType();
8954 QualType PtrTy = C.getPointerType(Ty).withRestrict();
8955 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
8956 auto *MapperVarDecl =
8957 cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl());
8958 SourceLocation Loc = D->getLocation();
8959 CharUnits ElementSize = C.getTypeSizeInChars(Ty);
8960
8961 // Prepare mapper function arguments and attributes.
8962 ImplicitParamDecl HandleArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
8963 C.VoidPtrTy, ImplicitParamDecl::Other);
8964 ImplicitParamDecl BaseArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
8965 ImplicitParamDecl::Other);
8966 ImplicitParamDecl BeginArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
8967 C.VoidPtrTy, ImplicitParamDecl::Other);
8968 ImplicitParamDecl SizeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty,
8969 ImplicitParamDecl::Other);
8970 ImplicitParamDecl TypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty,
8971 ImplicitParamDecl::Other);
8972 FunctionArgList Args;
8973 Args.push_back(&HandleArg);
8974 Args.push_back(&BaseArg);
8975 Args.push_back(&BeginArg);
8976 Args.push_back(&SizeArg);
8977 Args.push_back(&TypeArg);
8978 const CGFunctionInfo &FnInfo =
8979 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
8980 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
8981 SmallString<64> TyStr;
8982 llvm::raw_svector_ostream Out(TyStr);
8983 CGM.getCXXABI().getMangleContext().mangleTypeName(Ty, Out);
8984 std::string Name = getName({"omp_mapper", TyStr, D->getName()});
8985 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
8986 Name, &CGM.getModule());
8987 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
8988 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
8989 // Start the mapper function code generation.
8990 CodeGenFunction MapperCGF(CGM);
8991 MapperCGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
8992 // Compute the starting and end addreses of array elements.
8993 llvm::Value *Size = MapperCGF.EmitLoadOfScalar(
8994 MapperCGF.GetAddrOfLocalVar(&SizeArg), /*Volatile=*/false,
8995 C.getPointerType(Int64Ty), Loc);
8996 llvm::Value *PtrBegin = MapperCGF.Builder.CreateBitCast(
8997 MapperCGF.GetAddrOfLocalVar(&BeginArg).getPointer(),
8998 CGM.getTypes().ConvertTypeForMem(C.getPointerType(PtrTy)));
8999 llvm::Value *PtrEnd = MapperCGF.Builder.CreateGEP(PtrBegin, Size);
9000 llvm::Value *MapType = MapperCGF.EmitLoadOfScalar(
9001 MapperCGF.GetAddrOfLocalVar(&TypeArg), /*Volatile=*/false,
9002 C.getPointerType(Int64Ty), Loc);
9003 // Prepare common arguments for array initiation and deletion.
9004 llvm::Value *Handle = MapperCGF.EmitLoadOfScalar(
9005 MapperCGF.GetAddrOfLocalVar(&HandleArg),
9006 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
9007 llvm::Value *BaseIn = MapperCGF.EmitLoadOfScalar(
9008 MapperCGF.GetAddrOfLocalVar(&BaseArg),
9009 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
9010 llvm::Value *BeginIn = MapperCGF.EmitLoadOfScalar(
9011 MapperCGF.GetAddrOfLocalVar(&BeginArg),
9012 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
9013
9014 // Emit array initiation if this is an array section and \p MapType indicates
9015 // that memory allocation is required.
9016 llvm::BasicBlock *HeadBB = MapperCGF.createBasicBlock("omp.arraymap.head");
9017 emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType,
9018 ElementSize, HeadBB, /*IsInit=*/true);
9019
9020 // Emit a for loop to iterate through SizeArg of elements and map all of them.
9021
9022 // Emit the loop header block.
9023 MapperCGF.EmitBlock(HeadBB);
9024 llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.arraymap.body");
9025 llvm::BasicBlock *DoneBB = MapperCGF.createBasicBlock("omp.done");
9026 // Evaluate whether the initial condition is satisfied.
9027 llvm::Value *IsEmpty =
9028 MapperCGF.Builder.CreateICmpEQ(PtrBegin, PtrEnd, "omp.arraymap.isempty");
9029 MapperCGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
9030 llvm::BasicBlock *EntryBB = MapperCGF.Builder.GetInsertBlock();
9031
9032 // Emit the loop body block.
9033 MapperCGF.EmitBlock(BodyBB);
9034 llvm::PHINode *PtrPHI = MapperCGF.Builder.CreatePHI(
9035 PtrBegin->getType(), 2, "omp.arraymap.ptrcurrent");
9036 PtrPHI->addIncoming(PtrBegin, EntryBB);
9037 Address PtrCurrent =
9038 Address(PtrPHI, MapperCGF.GetAddrOfLocalVar(&BeginArg)
9039 .getAlignment()
9040 .alignmentOfArrayElement(ElementSize));
9041 // Privatize the declared variable of mapper to be the current array element.
9042 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
9043 Scope.addPrivate(MapperVarDecl, [&MapperCGF, PtrCurrent, PtrTy]() {
9044 return MapperCGF
9045 .EmitLoadOfPointerLValue(PtrCurrent, PtrTy->castAs<PointerType>())
9046 .getAddress();
9047 });
9048 (void)Scope.Privatize();
9049
9050 // Get map clause information. Fill up the arrays with all mapped variables.
9051 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9052 MappableExprsHandler::MapValuesArrayTy Pointers;
9053 MappableExprsHandler::MapValuesArrayTy Sizes;
9054 MappableExprsHandler::MapFlagsArrayTy MapTypes;
9055 MappableExprsHandler MEHandler(*D, MapperCGF);
9056 MEHandler.generateAllInfoForMapper(BasePointers, Pointers, Sizes, MapTypes);
9057
9058 // Call the runtime API __tgt_mapper_num_components to get the number of
9059 // pre-existing components.
9060 llvm::Value *OffloadingArgs[] = {Handle};
9061 llvm::Value *PreviousSize = MapperCGF.EmitRuntimeCall(
9062 createRuntimeFunction(OMPRTL__tgt_mapper_num_components), OffloadingArgs);
9063 llvm::Value *ShiftedPreviousSize = MapperCGF.Builder.CreateShl(
9064 PreviousSize,
9065 MapperCGF.Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
9066
9067 // Fill up the runtime mapper handle for all components.
9068 for (unsigned I = 0; I < BasePointers.size(); ++I) {
9069 llvm::Value *CurBaseArg = MapperCGF.Builder.CreateBitCast(
9070 *BasePointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
9071 llvm::Value *CurBeginArg = MapperCGF.Builder.CreateBitCast(
9072 Pointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
9073 llvm::Value *CurSizeArg = Sizes[I];
9074
9075 // Extract the MEMBER_OF field from the map type.
9076 llvm::BasicBlock *MemberBB = MapperCGF.createBasicBlock("omp.member");
9077 MapperCGF.EmitBlock(MemberBB);
9078 llvm::Value *OriMapType = MapperCGF.Builder.getInt64(MapTypes[I]);
9079 llvm::Value *Member = MapperCGF.Builder.CreateAnd(
9080 OriMapType,
9081 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_MEMBER_OF));
9082 llvm::BasicBlock *MemberCombineBB =
9083 MapperCGF.createBasicBlock("omp.member.combine");
9084 llvm::BasicBlock *TypeBB = MapperCGF.createBasicBlock("omp.type");
9085 llvm::Value *IsMember = MapperCGF.Builder.CreateIsNull(Member);
9086 MapperCGF.Builder.CreateCondBr(IsMember, TypeBB, MemberCombineBB);
9087 // Add the number of pre-existing components to the MEMBER_OF field if it
9088 // is valid.
9089 MapperCGF.EmitBlock(MemberCombineBB);
9090 llvm::Value *CombinedMember =
9091 MapperCGF.Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
9092 // Do nothing if it is not a member of previous components.
9093 MapperCGF.EmitBlock(TypeBB);
9094 llvm::PHINode *MemberMapType =
9095 MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.membermaptype");
9096 MemberMapType->addIncoming(OriMapType, MemberBB);
9097 MemberMapType->addIncoming(CombinedMember, MemberCombineBB);
9098
9099 // Combine the map type inherited from user-defined mapper with that
9100 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
9101 // bits of the \a MapType, which is the input argument of the mapper
9102 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
9103 // bits of MemberMapType.
9104 // [OpenMP 5.0], 1.2.6. map-type decay.
9105 // | alloc | to | from | tofrom | release | delete
9106 // ----------------------------------------------------------
9107 // alloc | alloc | alloc | alloc | alloc | release | delete
9108 // to | alloc | to | alloc | to | release | delete
9109 // from | alloc | alloc | from | from | release | delete
9110 // tofrom | alloc | to | from | tofrom | release | delete
9111 llvm::Value *LeftToFrom = MapperCGF.Builder.CreateAnd(
9112 MapType,
9113 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO |
9114 MappableExprsHandler::OMP_MAP_FROM));
9115 llvm::BasicBlock *AllocBB = MapperCGF.createBasicBlock("omp.type.alloc");
9116 llvm::BasicBlock *AllocElseBB =
9117 MapperCGF.createBasicBlock("omp.type.alloc.else");
9118 llvm::BasicBlock *ToBB = MapperCGF.createBasicBlock("omp.type.to");
9119 llvm::BasicBlock *ToElseBB = MapperCGF.createBasicBlock("omp.type.to.else");
9120 llvm::BasicBlock *FromBB = MapperCGF.createBasicBlock("omp.type.from");
9121 llvm::BasicBlock *EndBB = MapperCGF.createBasicBlock("omp.type.end");
9122 llvm::Value *IsAlloc = MapperCGF.Builder.CreateIsNull(LeftToFrom);
9123 MapperCGF.Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9124 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
9125 MapperCGF.EmitBlock(AllocBB);
9126 llvm::Value *AllocMapType = MapperCGF.Builder.CreateAnd(
9127 MemberMapType,
9128 MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9129 MappableExprsHandler::OMP_MAP_FROM)));
9130 MapperCGF.Builder.CreateBr(EndBB);
9131 MapperCGF.EmitBlock(AllocElseBB);
9132 llvm::Value *IsTo = MapperCGF.Builder.CreateICmpEQ(
9133 LeftToFrom,
9134 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO));
9135 MapperCGF.Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9136 // In case of to, clear OMP_MAP_FROM.
9137 MapperCGF.EmitBlock(ToBB);
9138 llvm::Value *ToMapType = MapperCGF.Builder.CreateAnd(
9139 MemberMapType,
9140 MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_FROM));
9141 MapperCGF.Builder.CreateBr(EndBB);
9142 MapperCGF.EmitBlock(ToElseBB);
9143 llvm::Value *IsFrom = MapperCGF.Builder.CreateICmpEQ(
9144 LeftToFrom,
9145 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_FROM));
9146 MapperCGF.Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9147 // In case of from, clear OMP_MAP_TO.
9148 MapperCGF.EmitBlock(FromBB);
9149 llvm::Value *FromMapType = MapperCGF.Builder.CreateAnd(
9150 MemberMapType,
9151 MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_TO));
9152 // In case of tofrom, do nothing.
9153 MapperCGF.EmitBlock(EndBB);
9154 llvm::PHINode *CurMapType =
9155 MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.maptype");
9156 CurMapType->addIncoming(AllocMapType, AllocBB);
9157 CurMapType->addIncoming(ToMapType, ToBB);
9158 CurMapType->addIncoming(FromMapType, FromBB);
9159 CurMapType->addIncoming(MemberMapType, ToElseBB);
9160
9161 // TODO: call the corresponding mapper function if a user-defined mapper is
9162 // associated with this map clause.
9163 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
9164 // data structure.
9165 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9166 CurSizeArg, CurMapType};
9167 MapperCGF.EmitRuntimeCall(
9168 createRuntimeFunction(OMPRTL__tgt_push_mapper_component),
9169 OffloadingArgs);
9170 }
9171
9172 // Update the pointer to point to the next element that needs to be mapped,
9173 // and check whether we have mapped all elements.
9174 llvm::Value *PtrNext = MapperCGF.Builder.CreateConstGEP1_32(
9175 PtrPHI, /*Idx0=*/1, "omp.arraymap.next");
9176 PtrPHI->addIncoming(PtrNext, BodyBB);
9177 llvm::Value *IsDone =
9178 MapperCGF.Builder.CreateICmpEQ(PtrNext, PtrEnd, "omp.arraymap.isdone");
9179 llvm::BasicBlock *ExitBB = MapperCGF.createBasicBlock("omp.arraymap.exit");
9180 MapperCGF.Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9181
9182 MapperCGF.EmitBlock(ExitBB);
9183 // Emit array deletion if this is an array section and \p MapType indicates
9184 // that deletion is required.
9185 emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType,
9186 ElementSize, DoneBB, /*IsInit=*/false);
9187
9188 // Emit the function exit block.
9189 MapperCGF.EmitBlock(DoneBB, /*IsFinished=*/true);
9190 MapperCGF.FinishFunction();
9191 UDMMap.try_emplace(D, Fn);
9192 if (CGF) {
9193 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn);
9194 Decls.second.push_back(D);
9195 }
9196}
9197
9198/// Emit the array initialization or deletion portion for user-defined mapper
9199/// code generation. First, it evaluates whether an array section is mapped and
9200/// whether the \a MapType instructs to delete this section. If \a IsInit is
9201/// true, and \a MapType indicates to not delete this array, array
9202/// initialization code is generated. If \a IsInit is false, and \a MapType
9203/// indicates to not this array, array deletion code is generated.
9204void CGOpenMPRuntime::emitUDMapperArrayInitOrDel(
9205 CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *Base,
9206 llvm::Value *Begin, llvm::Value *Size, llvm::Value *MapType,
9207 CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit) {
9208 StringRef Prefix = IsInit ? ".init" : ".del";
9209
9210 // Evaluate if this is an array section.
9211 llvm::BasicBlock *IsDeleteBB =
9212 MapperCGF.createBasicBlock("omp.array" + Prefix + ".evaldelete");
9213 llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.array" + Prefix);
9214 llvm::Value *IsArray = MapperCGF.Builder.CreateICmpSGE(
9215 Size, MapperCGF.Builder.getInt64(1), "omp.arrayinit.isarray");
9216 MapperCGF.Builder.CreateCondBr(IsArray, IsDeleteBB, ExitBB);
9217
9218 // Evaluate if we are going to delete this section.
9219 MapperCGF.EmitBlock(IsDeleteBB);
9220 llvm::Value *DeleteBit = MapperCGF.Builder.CreateAnd(
9221 MapType,
9222 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_DELETE));
9223 llvm::Value *DeleteCond;
9224 if (IsInit) {
9225 DeleteCond = MapperCGF.Builder.CreateIsNull(
9226 DeleteBit, "omp.array" + Prefix + ".delete");
9227 } else {
9228 DeleteCond = MapperCGF.Builder.CreateIsNotNull(
9229 DeleteBit, "omp.array" + Prefix + ".delete");
9230 }
9231 MapperCGF.Builder.CreateCondBr(DeleteCond, BodyBB, ExitBB);
9232
9233 MapperCGF.EmitBlock(BodyBB);
9234 // Get the array size by multiplying element size and element number (i.e., \p
9235 // Size).
9236 llvm::Value *ArraySize = MapperCGF.Builder.CreateNUWMul(
9237 Size, MapperCGF.Builder.getInt64(ElementSize.getQuantity()));
9238 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
9239 // memory allocation/deletion purpose only.
9240 llvm::Value *MapTypeArg = MapperCGF.Builder.CreateAnd(
9241 MapType,
9242 MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9243 MappableExprsHandler::OMP_MAP_FROM)));
9244 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
9245 // data structure.
9246 llvm::Value *OffloadingArgs[] = {Handle, Base, Begin, ArraySize, MapTypeArg};
9247 MapperCGF.EmitRuntimeCall(
9248 createRuntimeFunction(OMPRTL__tgt_push_mapper_component), OffloadingArgs);
9249}
9250
9251void CGOpenMPRuntime::emitTargetNumIterationsCall(
9252 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9253 llvm::Value *DeviceID,
9254 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9255 const OMPLoopDirective &D)>
9256 SizeEmitter) {
9257 OpenMPDirectiveKind Kind = D.getDirectiveKind();
9258 const OMPExecutableDirective *TD = &D;
9259 // Get nested teams distribute kind directive, if any.
9260 if (!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind))
9261 TD = getNestedDistributeDirective(CGM.getContext(), D);
9262 if (!TD)
9263 return;
9264 const auto *LD = cast<OMPLoopDirective>(TD);
9265 auto &&CodeGen = [LD, DeviceID, SizeEmitter, this](CodeGenFunction &CGF,
9266 PrePostActionTy &) {
9267 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) {
9268 llvm::Value *Args[] = {DeviceID, NumIterations};
9269 CGF.EmitRuntimeCall(
9270 createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args);
9271 }
9272 };
9273 emitInlinedDirective(CGF, OMPD_unknown, CodeGen);
9274}
9275
9276void CGOpenMPRuntime::emitTargetCall(
9277 CodeGenFunction &CGF, const OMPExecutableDirective &D,
9278 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
9279 const Expr *Device,
9280 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9281 const OMPLoopDirective &D)>
9282 SizeEmitter) {
9283 if (!CGF.HaveInsertPoint())
9284 return;
9285
9286 assert(OutlinedFn && "Invalid outlined function!")((OutlinedFn && "Invalid outlined function!") ? static_cast
<void> (0) : __assert_fail ("OutlinedFn && \"Invalid outlined function!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9286, __PRETTY_FUNCTION__))
;
9287
9288 const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>();
9289 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
9290 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
9291 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
9292 PrePostActionTy &) {
9293 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9294 };
9295 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
9296
9297 CodeGenFunction::OMPTargetDataInfo InputInfo;
9298 llvm::Value *MapTypesArray = nullptr;
9299 // Fill up the pointer arrays and transfer execution to the device.
9300 auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
9301 &MapTypesArray, &CS, RequiresOuterTask, &CapturedVars,
9302 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
9303 // On top of the arrays that were filled up, the target offloading call
9304 // takes as arguments the device id as well as the host pointer. The host
9305 // pointer is used by the runtime library to identify the current target
9306 // region, so it only has to be unique and not necessarily point to
9307 // anything. It could be the pointer to the outlined function that
9308 // implements the target region, but we aren't using that so that the
9309 // compiler doesn't need to keep that, and could therefore inline the host
9310 // function if proven worthwhile during optimization.
9311
9312 // From this point on, we need to have an ID of the target region defined.
9313 assert(OutlinedFnID && "Invalid outlined function ID!")((OutlinedFnID && "Invalid outlined function ID!") ? static_cast
<void> (0) : __assert_fail ("OutlinedFnID && \"Invalid outlined function ID!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9313, __PRETTY_FUNCTION__))
;
9314
9315 // Emit device ID if any.
9316 llvm::Value *DeviceID;
9317 if (Device) {
9318 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9319 CGF.Int64Ty, /*isSigned=*/true);
9320 } else {
9321 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9322 }
9323
9324 // Emit the number of elements in the offloading arrays.
9325 llvm::Value *PointerNum =
9326 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
9327
9328 // Return value of the runtime offloading call.
9329 llvm::Value *Return;
9330
9331 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D);
9332 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D);
9333
9334 // Emit tripcount for the target loop-based directive.
9335 emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter);
9336
9337 bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
9338 // The target region is an outlined function launched by the runtime
9339 // via calls __tgt_target() or __tgt_target_teams().
9340 //
9341 // __tgt_target() launches a target region with one team and one thread,
9342 // executing a serial region. This master thread may in turn launch
9343 // more threads within its team upon encountering a parallel region,
9344 // however, no additional teams can be launched on the device.
9345 //
9346 // __tgt_target_teams() launches a target region with one or more teams,
9347 // each with one or more threads. This call is required for target
9348 // constructs such as:
9349 // 'target teams'
9350 // 'target' / 'teams'
9351 // 'target teams distribute parallel for'
9352 // 'target parallel'
9353 // and so on.
9354 //
9355 // Note that on the host and CPU targets, the runtime implementation of
9356 // these calls simply call the outlined function without forking threads.
9357 // The outlined functions themselves have runtime calls to
9358 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
9359 // the compiler in emitTeamsCall() and emitParallelCall().
9360 //
9361 // In contrast, on the NVPTX target, the implementation of
9362 // __tgt_target_teams() launches a GPU kernel with the requested number
9363 // of teams and threads so no additional calls to the runtime are required.
9364 if (NumTeams) {
9365 // If we have NumTeams defined this means that we have an enclosed teams
9366 // region. Therefore we also expect to have NumThreads defined. These two
9367 // values should be defined in the presence of a teams directive,
9368 // regardless of having any clauses associated. If the user is using teams
9369 // but no clauses, these two values will be the default that should be
9370 // passed to the runtime library - a 32-bit integer with the value zero.
9371 assert(NumThreads && "Thread limit expression should be available along "((NumThreads && "Thread limit expression should be available along "
"with number of teams.") ? static_cast<void> (0) : __assert_fail
("NumThreads && \"Thread limit expression should be available along \" \"with number of teams.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9372, __PRETTY_FUNCTION__))
9372 "with number of teams.")((NumThreads && "Thread limit expression should be available along "
"with number of teams.") ? static_cast<void> (0) : __assert_fail
("NumThreads && \"Thread limit expression should be available along \" \"with number of teams.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9372, __PRETTY_FUNCTION__))
;
9373 llvm::Value *OffloadingArgs[] = {DeviceID,
9374 OutlinedFnID,
9375 PointerNum,
9376 InputInfo.BasePointersArray.getPointer(),
9377 InputInfo.PointersArray.getPointer(),
9378 InputInfo.SizesArray.getPointer(),
9379 MapTypesArray,
9380 NumTeams,
9381 NumThreads};
9382 Return = CGF.EmitRuntimeCall(
9383 createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_teams_nowait
9384 : OMPRTL__tgt_target_teams),
9385 OffloadingArgs);
9386 } else {
9387 llvm::Value *OffloadingArgs[] = {DeviceID,
9388 OutlinedFnID,
9389 PointerNum,
9390 InputInfo.BasePointersArray.getPointer(),
9391 InputInfo.PointersArray.getPointer(),
9392 InputInfo.SizesArray.getPointer(),
9393 MapTypesArray};
9394 Return = CGF.EmitRuntimeCall(
9395 createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_nowait
9396 : OMPRTL__tgt_target),
9397 OffloadingArgs);
9398 }
9399
9400 // Check the error code and execute the host version if required.
9401 llvm::BasicBlock *OffloadFailedBlock =
9402 CGF.createBasicBlock("omp_offload.failed");
9403 llvm::BasicBlock *OffloadContBlock =
9404 CGF.createBasicBlock("omp_offload.cont");
9405 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
9406 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
9407
9408 CGF.EmitBlock(OffloadFailedBlock);
9409 if (RequiresOuterTask) {
9410 CapturedVars.clear();
9411 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9412 }
9413 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
9414 CGF.EmitBranch(OffloadContBlock);
9415
9416 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
9417 };
9418
9419 // Notify that the host version must be executed.
9420 auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars,
9421 RequiresOuterTask](CodeGenFunction &CGF,
9422 PrePostActionTy &) {
9423 if (RequiresOuterTask) {
9424 CapturedVars.clear();
9425 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9426 }
9427 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
9428 };
9429
9430 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
9431 &CapturedVars, RequiresOuterTask,
9432 &CS](CodeGenFunction &CGF, PrePostActionTy &) {
9433 // Fill up the arrays with all the captured variables.
9434 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9435 MappableExprsHandler::MapValuesArrayTy Pointers;
9436 MappableExprsHandler::MapValuesArrayTy Sizes;
9437 MappableExprsHandler::MapFlagsArrayTy MapTypes;
9438
9439 // Get mappable expression information.
9440 MappableExprsHandler MEHandler(D, CGF);
9441 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9442
9443 auto RI = CS.getCapturedRecordDecl()->field_begin();
9444 auto CV = CapturedVars.begin();
9445 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
9446 CE = CS.capture_end();
9447 CI != CE; ++CI, ++RI, ++CV) {
9448 MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers;
9449 MappableExprsHandler::MapValuesArrayTy CurPointers;
9450 MappableExprsHandler::MapValuesArrayTy CurSizes;
9451 MappableExprsHandler::MapFlagsArrayTy CurMapTypes;
9452 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9453
9454 // VLA sizes are passed to the outlined region by copy and do not have map
9455 // information associated.
9456 if (CI->capturesVariableArrayType()) {
9457 CurBasePointers.push_back(*CV);
9458 CurPointers.push_back(*CV);
9459 CurSizes.push_back(CGF.Builder.CreateIntCast(
9460 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
9461 // Copy to the device as an argument. No need to retrieve it.
9462 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
9463 MappableExprsHandler::OMP_MAP_TARGET_PARAM |
9464 MappableExprsHandler::OMP_MAP_IMPLICIT);
9465 } else {
9466 // If we have any information in the map clause, we use it, otherwise we
9467 // just do a default mapping.
9468 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
9469 CurSizes, CurMapTypes, PartialStruct);
9470 if (CurBasePointers.empty())
9471 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
9472 CurPointers, CurSizes, CurMapTypes);
9473 // Generate correct mapping for variables captured by reference in
9474 // lambdas.
9475 if (CI->capturesVariable())
9476 MEHandler.generateInfoForLambdaCaptures(
9477 CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes,
9478 CurMapTypes, LambdaPointers);
9479 }
9480 // We expect to have at least an element of information for this capture.
9481 assert(!CurBasePointers.empty() &&((!CurBasePointers.empty() && "Non-existing map pointer for capture!"
) ? static_cast<void> (0) : __assert_fail ("!CurBasePointers.empty() && \"Non-existing map pointer for capture!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9482, __PRETTY_FUNCTION__))
9482 "Non-existing map pointer for capture!")((!CurBasePointers.empty() && "Non-existing map pointer for capture!"
) ? static_cast<void> (0) : __assert_fail ("!CurBasePointers.empty() && \"Non-existing map pointer for capture!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9482, __PRETTY_FUNCTION__))
;
9483 assert(CurBasePointers.size() == CurPointers.size() &&((CurBasePointers.size() == CurPointers.size() && CurBasePointers
.size() == CurSizes.size() && CurBasePointers.size() ==
CurMapTypes.size() && "Inconsistent map information sizes!"
) ? static_cast<void> (0) : __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9486, __PRETTY_FUNCTION__))
9484 CurBasePointers.size() == CurSizes.size() &&((CurBasePointers.size() == CurPointers.size() && CurBasePointers
.size() == CurSizes.size() && CurBasePointers.size() ==
CurMapTypes.size() && "Inconsistent map information sizes!"
) ? static_cast<void> (0) : __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9486, __PRETTY_FUNCTION__))
9485 CurBasePointers.size() == CurMapTypes.size() &&((CurBasePointers.size() == CurPointers.size() && CurBasePointers
.size() == CurSizes.size() && CurBasePointers.size() ==
CurMapTypes.size() && "Inconsistent map information sizes!"
) ? static_cast<void> (0) : __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9486, __PRETTY_FUNCTION__))
9486 "Inconsistent map information sizes!")((CurBasePointers.size() == CurPointers.size() && CurBasePointers
.size() == CurSizes.size() && CurBasePointers.size() ==
CurMapTypes.size() && "Inconsistent map information sizes!"
) ? static_cast<void> (0) : __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9486, __PRETTY_FUNCTION__))
;
9487
9488 // If there is an entry in PartialStruct it means we have a struct with
9489 // individual members mapped. Emit an extra combined entry.
9490 if (PartialStruct.Base.isValid())
9491 MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
9492 CurMapTypes, PartialStruct);
9493
9494 // We need to append the results of this capture to what we already have.
9495 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
9496 Pointers.append(CurPointers.begin(), CurPointers.end());
9497 Sizes.append(CurSizes.begin(), CurSizes.end());
9498 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
9499 }
9500 // Adjust MEMBER_OF flags for the lambdas captures.
9501 MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers,
9502 Pointers, MapTypes);
9503 // Map other list items in the map clause which are not captured variables
9504 // but "declare target link" global variables.
9505 MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
9506 MapTypes);
9507
9508 TargetDataInfo Info;
9509 // Fill up the arrays and create the arguments.
9510 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
9511 emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
9512 Info.PointersArray, Info.SizesArray,
9513 Info.MapTypesArray, Info);
9514 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9515 InputInfo.BasePointersArray =
9516 Address(Info.BasePointersArray, CGM.getPointerAlign());
9517 InputInfo.PointersArray =
9518 Address(Info.PointersArray, CGM.getPointerAlign());
9519 InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign());
9520 MapTypesArray = Info.MapTypesArray;
9521 if (RequiresOuterTask)
9522 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9523 else
9524 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
9525 };
9526
9527 auto &&TargetElseGen = [this, &ElseGen, &D, RequiresOuterTask](
9528 CodeGenFunction &CGF, PrePostActionTy &) {
9529 if (RequiresOuterTask) {
9530 CodeGenFunction::OMPTargetDataInfo InputInfo;
9531 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9532 } else {
9533 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
9534 }
9535 };
9536
9537 // If we have a target function ID it means that we need to support
9538 // offloading, otherwise, just execute on the host. We need to execute on host
9539 // regardless of the conditional in the if clause if, e.g., the user do not
9540 // specify target triples.
9541 if (OutlinedFnID) {
9542 if (IfCond) {
9543 emitOMPIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9544 } else {
9545 RegionCodeGenTy ThenRCG(TargetThenGen);
9546 ThenRCG(CGF);
9547 }
9548 } else {
9549 RegionCodeGenTy ElseRCG(TargetElseGen);
9550 ElseRCG(CGF);
9551 }
9552}
9553
9554void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
9555 StringRef ParentName) {
9556 if (!S)
9557 return;
9558
9559 // Codegen OMP target directives that offload compute to the device.
9560 bool RequiresDeviceCodegen =
9561 isa<OMPExecutableDirective>(S) &&
9562 isOpenMPTargetExecutionDirective(
9563 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9564
9565 if (RequiresDeviceCodegen) {
9566 const auto &E = *cast<OMPExecutableDirective>(S);
9567 unsigned DeviceID;
9568 unsigned FileID;
9569 unsigned Line;
9570 getTargetEntryUniqueInfo(CGM.getContext(), E.getBeginLoc(), DeviceID,
9571 FileID, Line);
9572
9573 // Is this a target region that should not be emitted as an entry point? If
9574 // so just signal we are done with this target region.
9575 if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
9576 ParentName, Line))
9577 return;
9578
9579 switch (E.getDirectiveKind()) {
9580 case OMPD_target:
9581 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
9582 cast<OMPTargetDirective>(E));
9583 break;
9584 case OMPD_target_parallel:
9585 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
9586 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9587 break;
9588 case OMPD_target_teams:
9589 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
9590 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9591 break;
9592 case OMPD_target_teams_distribute:
9593 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
9594 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9595 break;
9596 case OMPD_target_teams_distribute_simd:
9597 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
9598 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9599 break;
9600 case OMPD_target_parallel_for:
9601 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
9602 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9603 break;
9604 case OMPD_target_parallel_for_simd:
9605 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
9606 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9607 break;
9608 case OMPD_target_simd:
9609 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
9610 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9611 break;
9612 case OMPD_target_teams_distribute_parallel_for:
9613 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
9614 CGM, ParentName,
9615 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9616 break;
9617 case OMPD_target_teams_distribute_parallel_for_simd:
9618 CodeGenFunction::
9619 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
9620 CGM, ParentName,
9621 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9622 break;
9623 case OMPD_parallel:
9624 case OMPD_for:
9625 case OMPD_parallel_for:
9626 case OMPD_parallel_sections:
9627 case OMPD_for_simd:
9628 case OMPD_parallel_for_simd:
9629 case OMPD_cancel:
9630 case OMPD_cancellation_point:
9631 case OMPD_ordered:
9632 case OMPD_threadprivate:
9633 case OMPD_allocate:
9634 case OMPD_task:
9635 case OMPD_simd:
9636 case OMPD_sections:
9637 case OMPD_section:
9638 case OMPD_single:
9639 case OMPD_master:
9640 case OMPD_critical:
9641 case OMPD_taskyield:
9642 case OMPD_barrier:
9643 case OMPD_taskwait:
9644 case OMPD_taskgroup:
9645 case OMPD_atomic:
9646 case OMPD_flush:
9647 case OMPD_teams:
9648 case OMPD_target_data:
9649 case OMPD_target_exit_data:
9650 case OMPD_target_enter_data:
9651 case OMPD_distribute:
9652 case OMPD_distribute_simd:
9653 case OMPD_distribute_parallel_for:
9654 case OMPD_distribute_parallel_for_simd:
9655 case OMPD_teams_distribute:
9656 case OMPD_teams_distribute_simd:
9657 case OMPD_teams_distribute_parallel_for:
9658 case OMPD_teams_distribute_parallel_for_simd:
9659 case OMPD_target_update:
9660 case OMPD_declare_simd:
9661 case OMPD_declare_variant:
9662 case OMPD_declare_target:
9663 case OMPD_end_declare_target:
9664 case OMPD_declare_reduction:
9665 case OMPD_declare_mapper:
9666 case OMPD_taskloop:
9667 case OMPD_taskloop_simd:
9668 case OMPD_requires:
9669 case OMPD_unknown:
9670 llvm_unreachable("Unknown target directive for OpenMP device codegen.")::llvm::llvm_unreachable_internal("Unknown target directive for OpenMP device codegen."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9670)
;
9671 }
9672 return;
9673 }
9674
9675 if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9676 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9677 return;
9678
9679 scanForTargetRegionsFunctions(
9680 E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
9681 return;
9682 }
9683
9684 // If this is a lambda function, look into its body.
9685 if (const auto *L = dyn_cast<LambdaExpr>(S))
9686 S = L->getBody();
9687
9688 // Keep looking for target regions recursively.
9689 for (const Stmt *II : S->children())
9690 scanForTargetRegionsFunctions(II, ParentName);
9691}
9692
9693bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
9694 // If emitting code for the host, we do not process FD here. Instead we do
9695 // the normal code generation.
9696 if (!CGM.getLangOpts().OpenMPIsDevice) {
9697 if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) {
9698 Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9699 OMPDeclareTargetDeclAttr::getDeviceType(FD);
9700 // Do not emit device_type(nohost) functions for the host.
9701 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9702 return true;
9703 }
9704 return false;
9705 }
9706
9707 const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());
9708 StringRef Name = CGM.getMangledName(GD);
9709 // Try to detect target regions in the function.
9710 if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9711 scanForTargetRegionsFunctions(FD->getBody(), Name);
9712 Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9713 OMPDeclareTargetDeclAttr::getDeviceType(FD);
9714 // Do not emit device_type(nohost) functions for the host.
9715 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9716 return true;
9717 }
9718
9719 // Do not to emit function if it is not marked as declare target.
9720 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9721 AlreadyEmittedTargetFunctions.count(Name) == 0;
9722}
9723
9724bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
9725 if (!CGM.getLangOpts().OpenMPIsDevice)
9726 return false;
9727
9728 // Check if there are Ctors/Dtors in this declaration and look for target
9729 // regions in it. We use the complete variant to produce the kernel name
9730 // mangling.
9731 QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
9732 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
9733 for (const CXXConstructorDecl *Ctor : RD->ctors()) {
9734 StringRef ParentName =
9735 CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
9736 scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
9737 }
9738 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
9739 StringRef ParentName =
9740 CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
9741 scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
9742 }
9743 }
9744
9745 // Do not to emit variable if it is not marked as declare target.
9746 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9747 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9748 cast<VarDecl>(GD.getDecl()));
9749 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9750 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9751 HasRequiresUnifiedSharedMemory)) {
9752 DeferredGlobalVariables.insert(cast<VarDecl>(GD.getDecl()));
9753 return true;
9754 }
9755 return false;
9756}
9757
9758llvm::Constant *
9759CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF,
9760 const VarDecl *VD) {
9761 assert(VD->getType().isConstant(CGM.getContext()) &&((VD->getType().isConstant(CGM.getContext()) && "Expected constant variable."
) ? static_cast<void> (0) : __assert_fail ("VD->getType().isConstant(CGM.getContext()) && \"Expected constant variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9762, __PRETTY_FUNCTION__))
9762 "Expected constant variable.")((VD->getType().isConstant(CGM.getContext()) && "Expected constant variable."
) ? static_cast<void> (0) : __assert_fail ("VD->getType().isConstant(CGM.getContext()) && \"Expected constant variable.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9762, __PRETTY_FUNCTION__))
;
9763 StringRef VarName;
9764 llvm::Constant *Addr;
9765 llvm::GlobalValue::LinkageTypes Linkage;
9766 QualType Ty = VD->getType();
9767 SmallString<128> Buffer;
9768 {
9769 unsigned DeviceID;
9770 unsigned FileID;
9771 unsigned Line;
9772 getTargetEntryUniqueInfo(CGM.getContext(), VD->getLocation(), DeviceID,
9773 FileID, Line);
9774 llvm::raw_svector_ostream OS(Buffer);
9775 OS << "__omp_offloading_firstprivate_" << llvm::format("_%x", DeviceID)
9776 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
9777 VarName = OS.str();
9778 }
9779 Linkage = llvm::GlobalValue::InternalLinkage;
9780 Addr =
9781 getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(Ty), VarName,
9782 getDefaultFirstprivateAddressSpace());
9783 cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage);
9784 CharUnits VarSize = CGM.getContext().getTypeSizeInChars(Ty);
9785 CGM.addCompilerUsedGlobal(cast<llvm::GlobalValue>(Addr));
9786 OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
9787 VarName, Addr, VarSize,
9788 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo, Linkage);
9789 return Addr;
9790}
9791
9792void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
9793 llvm::Constant *Addr) {
9794 if (CGM.getLangOpts().OMPTargetTriples.empty())
9795 return;
9796 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9797 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9798 if (!Res) {
9799 if (CGM.getLangOpts().OpenMPIsDevice) {
9800 // Register non-target variables being emitted in device code (debug info
9801 // may cause this).
9802 StringRef VarName = CGM.getMangledName(VD);
9803 EmittedNonTargetVariables.try_emplace(VarName, Addr);
9804 }
9805 return;
9806 }
9807 // Register declare target variables.
9808 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags;
9809 StringRef VarName;
9810 CharUnits VarSize;
9811 llvm::GlobalValue::LinkageTypes Linkage;
9812
9813 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9814 !HasRequiresUnifiedSharedMemory) {
9815 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
9816 VarName = CGM.getMangledName(VD);
9817 if (VD->hasDefinition(CGM.getContext()) != VarDecl::DeclarationOnly) {
9818 VarSize = CGM.getContext().getTypeSizeInChars(VD->getType());
9819 assert(!VarSize.isZero() && "Expected non-zero size of the variable")((!VarSize.isZero() && "Expected non-zero size of the variable"
) ? static_cast<void> (0) : __assert_fail ("!VarSize.isZero() && \"Expected non-zero size of the variable\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9819, __PRETTY_FUNCTION__))
;
9820 } else {
9821 VarSize = CharUnits::Zero();
9822 }
9823 Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false);
9824 // Temp solution to prevent optimizations of the internal variables.
9825 if (CGM.getLangOpts().OpenMPIsDevice && !VD->isExternallyVisible()) {
9826 std::string RefName = getName({VarName, "ref"});
9827 if (!CGM.GetGlobalValue(RefName)) {
9828 llvm::Constant *AddrRef =
9829 getOrCreateInternalVariable(Addr->getType(), RefName);
9830 auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef);
9831 GVAddrRef->setConstant(/*Val=*/true);
9832 GVAddrRef->setLinkage(llvm::GlobalValue::InternalLinkage);
9833 GVAddrRef->setInitializer(Addr);
9834 CGM.addCompilerUsedGlobal(GVAddrRef);
9835 }
9836 }
9837 } else {
9838 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||((((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Declare target attribute must link or to with unified memory."
) ? static_cast<void> (0) : __assert_fail ("((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Declare target attribute must link or to with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9841, __PRETTY_FUNCTION__))
9839 (*Res == OMPDeclareTargetDeclAttr::MT_To &&((((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Declare target attribute must link or to with unified memory."
) ? static_cast<void> (0) : __assert_fail ("((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Declare target attribute must link or to with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9841, __PRETTY_FUNCTION__))
9840 HasRequiresUnifiedSharedMemory)) &&((((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Declare target attribute must link or to with unified memory."
) ? static_cast<void> (0) : __assert_fail ("((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Declare target attribute must link or to with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9841, __PRETTY_FUNCTION__))
9841 "Declare target attribute must link or to with unified memory.")((((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Declare target attribute must link or to with unified memory."
) ? static_cast<void> (0) : __assert_fail ("((*Res == OMPDeclareTargetDeclAttr::MT_Link) || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Declare target attribute must link or to with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9841, __PRETTY_FUNCTION__))
;
9842 if (*Res == OMPDeclareTargetDeclAttr::MT_Link)
9843 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink;
9844 else
9845 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
9846
9847 if (CGM.getLangOpts().OpenMPIsDevice) {
9848 VarName = Addr->getName();
9849 Addr = nullptr;
9850 } else {
9851 VarName = getAddrOfDeclareTargetVar(VD).getName();
9852 Addr = cast<llvm::Constant>(getAddrOfDeclareTargetVar(VD).getPointer());
9853 }
9854 VarSize = CGM.getPointerSize();
9855 Linkage = llvm::GlobalValue::WeakAnyLinkage;
9856 }
9857
9858 OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
9859 VarName, Addr, VarSize, Flags, Linkage);
9860}
9861
9862bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
9863 if (isa<FunctionDecl>(GD.getDecl()) ||
9864 isa<OMPDeclareReductionDecl>(GD.getDecl()))
9865 return emitTargetFunctions(GD);
9866
9867 return emitTargetGlobalVariable(GD);
9868}
9869
9870void CGOpenMPRuntime::emitDeferredTargetDecls() const {
9871 for (const VarDecl *VD : DeferredGlobalVariables) {
9872 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9873 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9874 if (!Res)
9875 continue;
9876 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9877 !HasRequiresUnifiedSharedMemory) {
9878 CGM.EmitGlobal(VD);
9879 } else {
9880 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||(((*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Expected link clause or to clause with unified memory.") ? static_cast
<void> (0) : __assert_fail ("(*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Expected link clause or to clause with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9883, __PRETTY_FUNCTION__))
9881 (*Res == OMPDeclareTargetDeclAttr::MT_To &&(((*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Expected link clause or to clause with unified memory.") ? static_cast
<void> (0) : __assert_fail ("(*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Expected link clause or to clause with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9883, __PRETTY_FUNCTION__))
9882 HasRequiresUnifiedSharedMemory)) &&(((*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Expected link clause or to clause with unified memory.") ? static_cast
<void> (0) : __assert_fail ("(*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Expected link clause or to clause with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9883, __PRETTY_FUNCTION__))
9883 "Expected link clause or to clause with unified memory.")(((*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr
::MT_To && HasRequiresUnifiedSharedMemory)) &&
"Expected link clause or to clause with unified memory.") ? static_cast
<void> (0) : __assert_fail ("(*Res == OMPDeclareTargetDeclAttr::MT_Link || (*Res == OMPDeclareTargetDeclAttr::MT_To && HasRequiresUnifiedSharedMemory)) && \"Expected link clause or to clause with unified memory.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9883, __PRETTY_FUNCTION__))
;
9884 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
9885 }
9886 }
9887}
9888
9889void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
9890 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
9891 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&((isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
" Expected target-based directive.") ? static_cast<void>
(0) : __assert_fail ("isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && \" Expected target-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9892, __PRETTY_FUNCTION__))
9892 " Expected target-based directive.")((isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
" Expected target-based directive.") ? static_cast<void>
(0) : __assert_fail ("isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && \" Expected target-based directive.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9892, __PRETTY_FUNCTION__))
;
9893}
9894
9895void CGOpenMPRuntime::checkArchForUnifiedAddressing(
9896 const OMPRequiresDecl *D) {
9897 for (const OMPClause *Clause : D->clauselists()) {
9898 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
9899 HasRequiresUnifiedSharedMemory = true;
9900 break;
9901 }
9902 }
9903}
9904
9905bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
9906 LangAS &AS) {
9907 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
9908 return false;
9909 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
9910 switch(A->getAllocatorType()) {
9911 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
9912 // Not supported, fallback to the default mem space.
9913 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
9914 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
9915 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
9916 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
9917 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
9918 case OMPAllocateDeclAttr::OMPConstMemAlloc:
9919 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
9920 AS = LangAS::Default;
9921 return true;
9922 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
9923 llvm_unreachable("Expected predefined allocator for the variables with the "::llvm::llvm_unreachable_internal("Expected predefined allocator for the variables with the "
"static storage.", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9924)
9924 "static storage.")::llvm::llvm_unreachable_internal("Expected predefined allocator for the variables with the "
"static storage.", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9924)
;
9925 }
9926 return false;
9927}
9928
9929bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
9930 return HasRequiresUnifiedSharedMemory;
9931}
9932
9933CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
9934 CodeGenModule &CGM)
9935 : CGM(CGM) {
9936 if (CGM.getLangOpts().OpenMPIsDevice) {
9937 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
9938 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
9939 }
9940}
9941
9942CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
9943 if (CGM.getLangOpts().OpenMPIsDevice)
9944 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
9945}
9946
9947bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
9948 if (!CGM.getLangOpts().OpenMPIsDevice || !ShouldMarkAsGlobal)
9949 return true;
9950
9951 StringRef Name = CGM.getMangledName(GD);
9952 const auto *D = cast<FunctionDecl>(GD.getDecl());
9953 // Do not to emit function if it is marked as declare target as it was already
9954 // emitted.
9955 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
9956 if (D->hasBody() && AlreadyEmittedTargetFunctions.count(Name) == 0) {
9957 if (auto *F = dyn_cast_or_null<llvm::Function>(CGM.GetGlobalValue(Name)))
9958 return !F->isDeclaration();
9959 return false;
9960 }
9961 return true;
9962 }
9963
9964 return !AlreadyEmittedTargetFunctions.insert(Name).second;
9965}
9966
9967llvm::Function *CGOpenMPRuntime::emitRequiresDirectiveRegFun() {
9968 // If we don't have entries or if we are emitting code for the device, we
9969 // don't need to do anything.
9970 if (CGM.getLangOpts().OMPTargetTriples.empty() ||
9971 CGM.getLangOpts().OpenMPSimd || CGM.getLangOpts().OpenMPIsDevice ||
9972 (OffloadEntriesInfoManager.empty() &&
9973 !HasEmittedDeclareTargetRegion &&
9974 !HasEmittedTargetRegion))
9975 return nullptr;
9976
9977 // Create and register the function that handles the requires directives.
9978 ASTContext &C = CGM.getContext();
9979
9980 llvm::Function *RequiresRegFn;
9981 {
9982 CodeGenFunction CGF(CGM);
9983 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
9984 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
9985 std::string ReqName = getName({"omp_offloading", "requires_reg"});
9986 RequiresRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, ReqName, FI);
9987 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {});
9988 OpenMPOffloadingRequiresDirFlags Flags = OMP_REQ_NONE;
9989 // TODO: check for other requires clauses.
9990 // The requires directive takes effect only when a target region is
9991 // present in the compilation unit. Otherwise it is ignored and not
9992 // passed to the runtime. This avoids the runtime from throwing an error
9993 // for mismatching requires clauses across compilation units that don't
9994 // contain at least 1 target region.
9995 assert((HasEmittedTargetRegion ||(((HasEmittedTargetRegion || HasEmittedDeclareTargetRegion ||
!OffloadEntriesInfoManager.empty()) && "Target or declare target region expected."
) ? static_cast<void> (0) : __assert_fail ("(HasEmittedTargetRegion || HasEmittedDeclareTargetRegion || !OffloadEntriesInfoManager.empty()) && \"Target or declare target region expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9998, __PRETTY_FUNCTION__))
9996 HasEmittedDeclareTargetRegion ||(((HasEmittedTargetRegion || HasEmittedDeclareTargetRegion ||
!OffloadEntriesInfoManager.empty()) && "Target or declare target region expected."
) ? static_cast<void> (0) : __assert_fail ("(HasEmittedTargetRegion || HasEmittedDeclareTargetRegion || !OffloadEntriesInfoManager.empty()) && \"Target or declare target region expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9998, __PRETTY_FUNCTION__))
9997 !OffloadEntriesInfoManager.empty()) &&(((HasEmittedTargetRegion || HasEmittedDeclareTargetRegion ||
!OffloadEntriesInfoManager.empty()) && "Target or declare target region expected."
) ? static_cast<void> (0) : __assert_fail ("(HasEmittedTargetRegion || HasEmittedDeclareTargetRegion || !OffloadEntriesInfoManager.empty()) && \"Target or declare target region expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9998, __PRETTY_FUNCTION__))
9998 "Target or declare target region expected.")(((HasEmittedTargetRegion || HasEmittedDeclareTargetRegion ||
!OffloadEntriesInfoManager.empty()) && "Target or declare target region expected."
) ? static_cast<void> (0) : __assert_fail ("(HasEmittedTargetRegion || HasEmittedDeclareTargetRegion || !OffloadEntriesInfoManager.empty()) && \"Target or declare target region expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 9998, __PRETTY_FUNCTION__))
;
9999 if (HasRequiresUnifiedSharedMemory)
10000 Flags = OMP_REQ_UNIFIED_SHARED_MEMORY;
10001 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_requires),
10002 llvm::ConstantInt::get(CGM.Int64Ty, Flags));
10003 CGF.FinishFunction();
10004 }
10005 return RequiresRegFn;
10006}
10007
10008llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() {
10009 // If we have offloading in the current module, we need to emit the entries
10010 // now and register the offloading descriptor.
10011 createOffloadEntriesAndInfoMetadata();
10012
10013 // Create and register the offloading binary descriptors. This is the main
10014 // entity that captures all the information about offloading in the current
10015 // compilation unit.
10016 return createOffloadingBinaryDescriptorRegistration();
10017}
10018
10019void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
10020 const OMPExecutableDirective &D,
10021 SourceLocation Loc,
10022 llvm::Function *OutlinedFn,
10023 ArrayRef<llvm::Value *> CapturedVars) {
10024 if (!CGF.HaveInsertPoint())
10025 return;
10026
10027 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
10028 CodeGenFunction::RunCleanupsScope Scope(CGF);
10029
10030 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
10031 llvm::Value *Args[] = {
10032 RTLoc,
10033 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
10034 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
10035 llvm::SmallVector<llvm::Value *, 16> RealArgs;
10036 RealArgs.append(std::begin(Args), std::end(Args));
10037 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10038
10039 llvm::FunctionCallee RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams);
10040 CGF.EmitRuntimeCall(RTLFn, RealArgs);
10041}
10042
10043void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
10044 const Expr *NumTeams,
10045 const Expr *ThreadLimit,
10046 SourceLocation Loc) {
10047 if (!CGF.HaveInsertPoint())
10048 return;
10049
10050 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
10051
10052 llvm::Value *NumTeamsVal =
10053 NumTeams
10054 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
10055 CGF.CGM.Int32Ty, /* isSigned = */ true)
10056 : CGF.Builder.getInt32(0);
10057
10058 llvm::Value *ThreadLimitVal =
10059 ThreadLimit
10060 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
10061 CGF.CGM.Int32Ty, /* isSigned = */ true)
10062 : CGF.Builder.getInt32(0);
10063
10064 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
10065 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
10066 ThreadLimitVal};
10067 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
10068 PushNumTeamsArgs);
10069}
10070
10071void CGOpenMPRuntime::emitTargetDataCalls(
10072 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10073 const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
10074 if (!CGF.HaveInsertPoint())
10075 return;
10076
10077 // Action used to replace the default codegen action and turn privatization
10078 // off.
10079 PrePostActionTy NoPrivAction;
10080
10081 // Generate the code for the opening of the data environment. Capture all the
10082 // arguments of the runtime call by reference because they are used in the
10083 // closing of the region.
10084 auto &&BeginThenGen = [this, &D, Device, &Info,
10085 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
10086 // Fill up the arrays with all the mapped variables.
10087 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
10088 MappableExprsHandler::MapValuesArrayTy Pointers;
10089 MappableExprsHandler::MapValuesArrayTy Sizes;
10090 MappableExprsHandler::MapFlagsArrayTy MapTypes;
10091
10092 // Get map clause information.
10093 MappableExprsHandler MCHandler(D, CGF);
10094 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
10095
10096 // Fill up the arrays and create the arguments.
10097 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
10098
10099 llvm::Value *BasePointersArrayArg = nullptr;
10100 llvm::Value *PointersArrayArg = nullptr;
10101 llvm::Value *SizesArrayArg = nullptr;
10102 llvm::Value *MapTypesArrayArg = nullptr;
10103 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
10104 SizesArrayArg, MapTypesArrayArg, Info);
10105
10106 // Emit device ID if any.
10107 llvm::Value *DeviceID = nullptr;
10108 if (Device) {
10109 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10110 CGF.Int64Ty, /*isSigned=*/true);
10111 } else {
10112 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10113 }
10114
10115 // Emit the number of elements in the offloading arrays.
10116 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
10117
10118 llvm::Value *OffloadingArgs[] = {
10119 DeviceID, PointerNum, BasePointersArrayArg,
10120 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
10121 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin),
10122 OffloadingArgs);
10123
10124 // If device pointer privatization is required, emit the body of the region
10125 // here. It will have to be duplicated: with and without privatization.
10126 if (!Info.CaptureDeviceAddrMap.empty())
10127 CodeGen(CGF);
10128 };
10129
10130 // Generate code for the closing of the data region.
10131 auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF,
10132 PrePostActionTy &) {
10133 assert(Info.isValid() && "Invalid data environment closing arguments.")((Info.isValid() && "Invalid data environment closing arguments."
) ? static_cast<void> (0) : __assert_fail ("Info.isValid() && \"Invalid data environment closing arguments.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10133, __PRETTY_FUNCTION__))
;
10134
10135 llvm::Value *BasePointersArrayArg = nullptr;
10136 llvm::Value *PointersArrayArg = nullptr;
10137 llvm::Value *SizesArrayArg = nullptr;
10138 llvm::Value *MapTypesArrayArg = nullptr;
10139 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
10140 SizesArrayArg, MapTypesArrayArg, Info);
10141
10142 // Emit device ID if any.
10143 llvm::Value *DeviceID = nullptr;
10144 if (Device) {
10145 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10146 CGF.Int64Ty, /*isSigned=*/true);
10147 } else {
10148 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10149 }
10150
10151 // Emit the number of elements in the offloading arrays.
10152 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
10153
10154 llvm::Value *OffloadingArgs[] = {
10155 DeviceID, PointerNum, BasePointersArrayArg,
10156 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
10157 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end),
10158 OffloadingArgs);
10159 };
10160
10161 // If we need device pointer privatization, we need to emit the body of the
10162 // region with no privatization in the 'else' branch of the conditional.
10163 // Otherwise, we don't have to do anything.
10164 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF,
10165 PrePostActionTy &) {
10166 if (!Info.CaptureDeviceAddrMap.empty()) {
10167 CodeGen.setAction(NoPrivAction);
10168 CodeGen(CGF);
10169 }
10170 };
10171
10172 // We don't have to do anything to close the region if the if clause evaluates
10173 // to false.
10174 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
10175
10176 if (IfCond) {
10177 emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
10178 } else {
10179 RegionCodeGenTy RCG(BeginThenGen);
10180 RCG(CGF);
10181 }
10182
10183 // If we don't require privatization of device pointers, we emit the body in
10184 // between the runtime calls. This avoids duplicating the body code.
10185 if (Info.CaptureDeviceAddrMap.empty()) {
10186 CodeGen.setAction(NoPrivAction);
10187 CodeGen(CGF);
10188 }
10189
10190 if (IfCond) {
10191 emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen);
10192 } else {
10193 RegionCodeGenTy RCG(EndThenGen);
10194 RCG(CGF);
10195 }
10196}
10197
10198void CGOpenMPRuntime::emitTargetDataStandAloneCall(
10199 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10200 const Expr *Device) {
10201 if (!CGF.HaveInsertPoint())
10202 return;
10203
10204 assert((isa<OMPTargetEnterDataDirective>(D) ||(((isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective
>(D) || isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives."
) ? static_cast<void> (0) : __assert_fail ("(isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective>(D) || isa<OMPTargetUpdateDirective>(D)) && \"Expecting either target enter, exit data, or update directives.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10207, __PRETTY_FUNCTION__))
10205 isa<OMPTargetExitDataDirective>(D) ||(((isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective
>(D) || isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives."
) ? static_cast<void> (0) : __assert_fail ("(isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective>(D) || isa<OMPTargetUpdateDirective>(D)) && \"Expecting either target enter, exit data, or update directives.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10207, __PRETTY_FUNCTION__))
10206 isa<OMPTargetUpdateDirective>(D)) &&(((isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective
>(D) || isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives."
) ? static_cast<void> (0) : __assert_fail ("(isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective>(D) || isa<OMPTargetUpdateDirective>(D)) && \"Expecting either target enter, exit data, or update directives.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10207, __PRETTY_FUNCTION__))
10207 "Expecting either target enter, exit data, or update directives.")(((isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective
>(D) || isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives."
) ? static_cast<void> (0) : __assert_fail ("(isa<OMPTargetEnterDataDirective>(D) || isa<OMPTargetExitDataDirective>(D) || isa<OMPTargetUpdateDirective>(D)) && \"Expecting either target enter, exit data, or update directives.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10207, __PRETTY_FUNCTION__))
;
10208
10209 CodeGenFunction::OMPTargetDataInfo InputInfo;
10210 llvm::Value *MapTypesArray = nullptr;
10211 // Generate the code for the opening of the data environment.
10212 auto &&ThenGen = [this, &D, Device, &InputInfo,
10213 &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) {
10214 // Emit device ID if any.
10215 llvm::Value *DeviceID = nullptr;
10216 if (Device) {
10217 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10218 CGF.Int64Ty, /*isSigned=*/true);
10219 } else {
10220 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10221 }
10222
10223 // Emit the number of elements in the offloading arrays.
10224 llvm::Constant *PointerNum =
10225 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
10226
10227 llvm::Value *OffloadingArgs[] = {DeviceID,
10228 PointerNum,
10229 InputInfo.BasePointersArray.getPointer(),
10230 InputInfo.PointersArray.getPointer(),
10231 InputInfo.SizesArray.getPointer(),
10232 MapTypesArray};
10233
10234 // Select the right runtime function call for each expected standalone
10235 // directive.
10236 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
10237 OpenMPRTLFunction RTLFn;
10238 switch (D.getDirectiveKind()) {
10239 case OMPD_target_enter_data:
10240 RTLFn = HasNowait ? OMPRTL__tgt_target_data_begin_nowait
10241 : OMPRTL__tgt_target_data_begin;
10242 break;
10243 case OMPD_target_exit_data:
10244 RTLFn = HasNowait ? OMPRTL__tgt_target_data_end_nowait
10245 : OMPRTL__tgt_target_data_end;
10246 break;
10247 case OMPD_target_update:
10248 RTLFn = HasNowait ? OMPRTL__tgt_target_data_update_nowait
10249 : OMPRTL__tgt_target_data_update;
10250 break;
10251 case OMPD_parallel:
10252 case OMPD_for:
10253 case OMPD_parallel_for:
10254 case OMPD_parallel_sections:
10255 case OMPD_for_simd:
10256 case OMPD_parallel_for_simd:
10257 case OMPD_cancel:
10258 case OMPD_cancellation_point:
10259 case OMPD_ordered:
10260 case OMPD_threadprivate:
10261 case OMPD_allocate:
10262 case OMPD_task:
10263 case OMPD_simd:
10264 case OMPD_sections:
10265 case OMPD_section:
10266 case OMPD_single:
10267 case OMPD_master:
10268 case OMPD_critical:
10269 case OMPD_taskyield:
10270 case OMPD_barrier:
10271 case OMPD_taskwait:
10272 case OMPD_taskgroup:
10273 case OMPD_atomic:
10274 case OMPD_flush:
10275 case OMPD_teams:
10276 case OMPD_target_data:
10277 case OMPD_distribute:
10278 case OMPD_distribute_simd:
10279 case OMPD_distribute_parallel_for:
10280 case OMPD_distribute_parallel_for_simd:
10281 case OMPD_teams_distribute:
10282 case OMPD_teams_distribute_simd:
10283 case OMPD_teams_distribute_parallel_for:
10284 case OMPD_teams_distribute_parallel_for_simd:
10285 case OMPD_declare_simd:
10286 case OMPD_declare_variant:
10287 case OMPD_declare_target:
10288 case OMPD_end_declare_target:
10289 case OMPD_declare_reduction:
10290 case OMPD_declare_mapper:
10291 case OMPD_taskloop:
10292 case OMPD_taskloop_simd:
10293 case OMPD_target:
10294 case OMPD_target_simd:
10295 case OMPD_target_teams_distribute:
10296 case OMPD_target_teams_distribute_simd:
10297 case OMPD_target_teams_distribute_parallel_for:
10298 case OMPD_target_teams_distribute_parallel_for_simd:
10299 case OMPD_target_teams:
10300 case OMPD_target_parallel:
10301 case OMPD_target_parallel_for:
10302 case OMPD_target_parallel_for_simd:
10303 case OMPD_requires:
10304 case OMPD_unknown:
10305 llvm_unreachable("Unexpected standalone target data directive.")::llvm::llvm_unreachable_internal("Unexpected standalone target data directive."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10305)
;
10306 break;
10307 }
10308 CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs);
10309 };
10310
10311 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
10312 CodeGenFunction &CGF, PrePostActionTy &) {
10313 // Fill up the arrays with all the mapped variables.
10314 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
10315 MappableExprsHandler::MapValuesArrayTy Pointers;
10316 MappableExprsHandler::MapValuesArrayTy Sizes;
10317 MappableExprsHandler::MapFlagsArrayTy MapTypes;
10318
10319 // Get map clause information.
10320 MappableExprsHandler MEHandler(D, CGF);
10321 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
10322
10323 TargetDataInfo Info;
10324 // Fill up the arrays and create the arguments.
10325 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
10326 emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
10327 Info.PointersArray, Info.SizesArray,
10328 Info.MapTypesArray, Info);
10329 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
10330 InputInfo.BasePointersArray =
10331 Address(Info.BasePointersArray, CGM.getPointerAlign());
10332 InputInfo.PointersArray =
10333 Address(Info.PointersArray, CGM.getPointerAlign());
10334 InputInfo.SizesArray =
10335 Address(Info.SizesArray, CGM.getPointerAlign());
10336 MapTypesArray = Info.MapTypesArray;
10337 if (D.hasClausesOfKind<OMPDependClause>())
10338 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
10339 else
10340 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
10341 };
10342
10343 if (IfCond) {
10344 emitOMPIfClause(CGF, IfCond, TargetThenGen,
10345 [](CodeGenFunction &CGF, PrePostActionTy &) {});
10346 } else {
10347 RegionCodeGenTy ThenRCG(TargetThenGen);
10348 ThenRCG(CGF);
10349 }
10350}
10351
10352namespace {
10353 /// Kind of parameter in a function with 'declare simd' directive.
10354 enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
10355 /// Attribute set of the parameter.
10356 struct ParamAttrTy {
10357 ParamKindTy Kind = Vector;
10358 llvm::APSInt StrideOrArg;
10359 llvm::APSInt Alignment;
10360 };
10361} // namespace
10362
10363static unsigned evaluateCDTSize(const FunctionDecl *FD,
10364 ArrayRef<ParamAttrTy> ParamAttrs) {
10365 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
10366 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
10367 // of that clause. The VLEN value must be power of 2.
10368 // In other case the notion of the function`s "characteristic data type" (CDT)
10369 // is used to compute the vector length.
10370 // CDT is defined in the following order:
10371 // a) For non-void function, the CDT is the return type.
10372 // b) If the function has any non-uniform, non-linear parameters, then the
10373 // CDT is the type of the first such parameter.
10374 // c) If the CDT determined by a) or b) above is struct, union, or class
10375 // type which is pass-by-value (except for the type that maps to the
10376 // built-in complex data type), the characteristic data type is int.
10377 // d) If none of the above three cases is applicable, the CDT is int.
10378 // The VLEN is then determined based on the CDT and the size of vector
10379 // register of that ISA for which current vector version is generated. The
10380 // VLEN is computed using the formula below:
10381 // VLEN = sizeof(vector_register) / sizeof(CDT),
10382 // where vector register size specified in section 3.2.1 Registers and the
10383 // Stack Frame of original AMD64 ABI document.
10384 QualType RetType = FD->getReturnType();
10385 if (RetType.isNull())
10386 return 0;
10387 ASTContext &C = FD->getASTContext();
10388 QualType CDT;
10389 if (!RetType.isNull() && !RetType->isVoidType()) {
10390 CDT = RetType;
10391 } else {
10392 unsigned Offset = 0;
10393 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10394 if (ParamAttrs[Offset].Kind == Vector)
10395 CDT = C.getPointerType(C.getRecordType(MD->getParent()));
10396 ++Offset;
10397 }
10398 if (CDT.isNull()) {
10399 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
10400 if (ParamAttrs[I + Offset].Kind == Vector) {
10401 CDT = FD->getParamDecl(I)->getType();
10402 break;
10403 }
10404 }
10405 }
10406 }
10407 if (CDT.isNull())
10408 CDT = C.IntTy;
10409 CDT = CDT->getCanonicalTypeUnqualified();
10410 if (CDT->isRecordType() || CDT->isUnionType())
10411 CDT = C.IntTy;
10412 return C.getTypeSize(CDT);
10413}
10414
10415static void
10416emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
10417 const llvm::APSInt &VLENVal,
10418 ArrayRef<ParamAttrTy> ParamAttrs,
10419 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10420 struct ISADataTy {
10421 char ISA;
10422 unsigned VecRegSize;
10423 };
10424 ISADataTy ISAData[] = {
10425 {
10426 'b', 128
10427 }, // SSE
10428 {
10429 'c', 256
10430 }, // AVX
10431 {
10432 'd', 256
10433 }, // AVX2
10434 {
10435 'e', 512
10436 }, // AVX512
10437 };
10438 llvm::SmallVector<char, 2> Masked;
10439 switch (State) {
10440 case OMPDeclareSimdDeclAttr::BS_Undefined:
10441 Masked.push_back('N');
10442 Masked.push_back('M');
10443 break;
10444 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10445 Masked.push_back('N');
10446 break;
10447 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10448 Masked.push_back('M');
10449 break;
10450 }
10451 for (char Mask : Masked) {
10452 for (const ISADataTy &Data : ISAData) {
10453 SmallString<256> Buffer;
10454 llvm::raw_svector_ostream Out(Buffer);
10455 Out << "_ZGV" << Data.ISA << Mask;
10456 if (!VLENVal) {
10457 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
10458 assert(NumElts && "Non-zero simdlen/cdtsize expected")((NumElts && "Non-zero simdlen/cdtsize expected") ? static_cast
<void> (0) : __assert_fail ("NumElts && \"Non-zero simdlen/cdtsize expected\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10458, __PRETTY_FUNCTION__))
;
10459 Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts);
10460 } else {
10461 Out << VLENVal;
10462 }
10463 for (const ParamAttrTy &ParamAttr : ParamAttrs) {
10464 switch (ParamAttr.Kind){
10465 case LinearWithVarStride:
10466 Out << 's' << ParamAttr.StrideOrArg;
10467 break;
10468 case Linear:
10469 Out << 'l';
10470 if (!!ParamAttr.StrideOrArg)
10471 Out << ParamAttr.StrideOrArg;
10472 break;
10473 case Uniform:
10474 Out << 'u';
10475 break;
10476 case Vector:
10477 Out << 'v';
10478 break;
10479 }
10480 if (!!ParamAttr.Alignment)
10481 Out << 'a' << ParamAttr.Alignment;
10482 }
10483 Out << '_' << Fn->getName();
10484 Fn->addFnAttr(Out.str());
10485 }
10486 }
10487}
10488
10489// This are the Functions that are needed to mangle the name of the
10490// vector functions generated by the compiler, according to the rules
10491// defined in the "Vector Function ABI specifications for AArch64",
10492// available at
10493// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
10494
10495/// Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
10496///
10497/// TODO: Need to implement the behavior for reference marked with a
10498/// var or no linear modifiers (1.b in the section). For this, we
10499/// need to extend ParamKindTy to support the linear modifiers.
10500static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {
10501 QT = QT.getCanonicalType();
10502
10503 if (QT->isVoidType())
10504 return false;
10505
10506 if (Kind == ParamKindTy::Uniform)
10507 return false;
10508
10509 if (Kind == ParamKindTy::Linear)
10510 return false;
10511
10512 // TODO: Handle linear references with modifiers
10513
10514 if (Kind == ParamKindTy::LinearWithVarStride)
10515 return false;
10516
10517 return true;
10518}
10519
10520/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
10521static bool getAArch64PBV(QualType QT, ASTContext &C) {
10522 QT = QT.getCanonicalType();
10523 unsigned Size = C.getTypeSize(QT);
10524
10525 // Only scalars and complex within 16 bytes wide set PVB to true.
10526 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10527 return false;
10528
10529 if (QT->isFloatingType())
10530 return true;
10531
10532 if (QT->isIntegerType())
10533 return true;
10534
10535 if (QT->isPointerType())
10536 return true;
10537
10538 // TODO: Add support for complex types (section 3.1.2, item 2).
10539
10540 return false;
10541}
10542
10543/// Computes the lane size (LS) of a return type or of an input parameter,
10544/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
10545/// TODO: Add support for references, section 3.2.1, item 1.
10546static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) {
10547 if (getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
10548 QualType PTy = QT.getCanonicalType()->getPointeeType();
10549 if (getAArch64PBV(PTy, C))
10550 return C.getTypeSize(PTy);
10551 }
10552 if (getAArch64PBV(QT, C))
10553 return C.getTypeSize(QT);
10554
10555 return C.getTypeSize(C.getUIntPtrType());
10556}
10557
10558// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
10559// signature of the scalar function, as defined in 3.2.2 of the
10560// AAVFABI.
10561static std::tuple<unsigned, unsigned, bool>
10562getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) {
10563 QualType RetType = FD->getReturnType().getCanonicalType();
10564
10565 ASTContext &C = FD->getASTContext();
10566
10567 bool OutputBecomesInput = false;
10568
10569 llvm::SmallVector<unsigned, 8> Sizes;
10570 if (!RetType->isVoidType()) {
10571 Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C));
10572 if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {}))
10573 OutputBecomesInput = true;
10574 }
10575 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
10576 QualType QT = FD->getParamDecl(I)->getType().getCanonicalType();
10577 Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C));
10578 }
10579
10580 assert(!Sizes.empty() && "Unable to determine NDS and WDS.")((!Sizes.empty() && "Unable to determine NDS and WDS."
) ? static_cast<void> (0) : __assert_fail ("!Sizes.empty() && \"Unable to determine NDS and WDS.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10580, __PRETTY_FUNCTION__))
;
10581 // The LS of a function parameter / return value can only be a power
10582 // of 2, starting from 8 bits, up to 128.
10583 assert(std::all_of(Sizes.begin(), Sizes.end(),((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
10584 [](unsigned Size) {((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
10585 return Size == 8 || Size == 16 || Size == 32 ||((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
10586 Size == 64 || Size == 128;((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
10587 }) &&((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
10588 "Invalid size")((std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) {
return Size == 8 || Size == 16 || Size == 32 || Size == 64 ||
Size == 128; }) && "Invalid size") ? static_cast<
void> (0) : __assert_fail ("std::all_of(Sizes.begin(), Sizes.end(), [](unsigned Size) { return Size == 8 || Size == 16 || Size == 32 || Size == 64 || Size == 128; }) && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10588, __PRETTY_FUNCTION__))
;
10589
10590 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10591 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10592 OutputBecomesInput);
10593}
10594
10595/// Mangle the parameter part of the vector function name according to
10596/// their OpenMP classification. The mangling function is defined in
10597/// section 3.5 of the AAVFABI.
10598static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {
10599 SmallString<256> Buffer;
10600 llvm::raw_svector_ostream Out(Buffer);
10601 for (const auto &ParamAttr : ParamAttrs) {
10602 switch (ParamAttr.Kind) {
10603 case LinearWithVarStride:
10604 Out << "ls" << ParamAttr.StrideOrArg;
10605 break;
10606 case Linear:
10607 Out << 'l';
10608 // Don't print the step value if it is not present or if it is
10609 // equal to 1.
10610 if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1)
10611 Out << ParamAttr.StrideOrArg;
10612 break;
10613 case Uniform:
10614 Out << 'u';
10615 break;
10616 case Vector:
10617 Out << 'v';
10618 break;
10619 }
10620
10621 if (!!ParamAttr.Alignment)
10622 Out << 'a' << ParamAttr.Alignment;
10623 }
10624
10625 return Out.str();
10626}
10627
10628// Function used to add the attribute. The parameter `VLEN` is
10629// templated to allow the use of "x" when targeting scalable functions
10630// for SVE.
10631template <typename T>
10632static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
10633 char ISA, StringRef ParSeq,
10634 StringRef MangledName, bool OutputBecomesInput,
10635 llvm::Function *Fn) {
10636 SmallString<256> Buffer;
10637 llvm::raw_svector_ostream Out(Buffer);
10638 Out << Prefix << ISA << LMask << VLEN;
10639 if (OutputBecomesInput)
10640 Out << "v";
10641 Out << ParSeq << "_" << MangledName;
10642 Fn->addFnAttr(Out.str());
10643}
10644
10645// Helper function to generate the Advanced SIMD names depending on
10646// the value of the NDS when simdlen is not present.
10647static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
10648 StringRef Prefix, char ISA,
10649 StringRef ParSeq, StringRef MangledName,
10650 bool OutputBecomesInput,
10651 llvm::Function *Fn) {
10652 switch (NDS) {
10653 case 8:
10654 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
10655 OutputBecomesInput, Fn);
10656 addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName,
10657 OutputBecomesInput, Fn);
10658 break;
10659 case 16:
10660 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
10661 OutputBecomesInput, Fn);
10662 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
10663 OutputBecomesInput, Fn);
10664 break;
10665 case 32:
10666 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
10667 OutputBecomesInput, Fn);
10668 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
10669 OutputBecomesInput, Fn);
10670 break;
10671 case 64:
10672 case 128:
10673 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
10674 OutputBecomesInput, Fn);
10675 break;
10676 default:
10677 llvm_unreachable("Scalar type is too wide.")::llvm::llvm_unreachable_internal("Scalar type is too wide.",
"/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10677)
;
10678 }
10679}
10680
10681/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
10682static void emitAArch64DeclareSimdFunction(
10683 CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN,
10684 ArrayRef<ParamAttrTy> ParamAttrs,
10685 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10686 char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) {
10687
10688 // Get basic data for building the vector signature.
10689 const auto Data = getNDSWDS(FD, ParamAttrs);
10690 const unsigned NDS = std::get<0>(Data);
10691 const unsigned WDS = std::get<1>(Data);
10692 const bool OutputBecomesInput = std::get<2>(Data);
10693
10694 // Check the values provided via `simdlen` by the user.
10695 // 1. A `simdlen(1)` doesn't produce vector signatures,
10696 if (UserVLEN == 1) {
10697 unsigned DiagID = CGM.getDiags().getCustomDiagID(
10698 DiagnosticsEngine::Warning,
10699 "The clause simdlen(1) has no effect when targeting aarch64.");
10700 CGM.getDiags().Report(SLoc, DiagID);
10701 return;
10702 }
10703
10704 // 2. Section 3.3.1, item 1: user input must be a power of 2 for
10705 // Advanced SIMD output.
10706 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10707 unsigned DiagID = CGM.getDiags().getCustomDiagID(
10708 DiagnosticsEngine::Warning, "The value specified in simdlen must be a "
10709 "power of 2 when targeting Advanced SIMD.");
10710 CGM.getDiags().Report(SLoc, DiagID);
10711 return;
10712 }
10713
10714 // 3. Section 3.4.1. SVE fixed lengh must obey the architectural
10715 // limits.
10716 if (ISA == 's' && UserVLEN != 0) {
10717 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10718 unsigned DiagID = CGM.getDiags().getCustomDiagID(
10719 DiagnosticsEngine::Warning, "The clause simdlen must fit the %0-bit "
10720 "lanes in the architectural constraints "
10721 "for SVE (min is 128-bit, max is "
10722 "2048-bit, by steps of 128-bit)");
10723 CGM.getDiags().Report(SLoc, DiagID) << WDS;
10724 return;
10725 }
10726 }
10727
10728 // Sort out parameter sequence.
10729 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
10730 StringRef Prefix = "_ZGV";
10731 // Generate simdlen from user input (if any).
10732 if (UserVLEN) {
10733 if (ISA == 's') {
10734 // SVE generates only a masked function.
10735 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10736 OutputBecomesInput, Fn);
10737 } else {
10738 assert(ISA == 'n' && "Expected ISA either 's' or 'n'.")((ISA == 'n' && "Expected ISA either 's' or 'n'.") ? static_cast
<void> (0) : __assert_fail ("ISA == 'n' && \"Expected ISA either 's' or 'n'.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10738, __PRETTY_FUNCTION__))
;
10739 // Advanced SIMD generates one or two functions, depending on
10740 // the `[not]inbranch` clause.
10741 switch (State) {
10742 case OMPDeclareSimdDeclAttr::BS_Undefined:
10743 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
10744 OutputBecomesInput, Fn);
10745 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10746 OutputBecomesInput, Fn);
10747 break;
10748 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10749 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
10750 OutputBecomesInput, Fn);
10751 break;
10752 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10753 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10754 OutputBecomesInput, Fn);
10755 break;
10756 }
10757 }
10758 } else {
10759 // If no user simdlen is provided, follow the AAVFABI rules for
10760 // generating the vector length.
10761 if (ISA == 's') {
10762 // SVE, section 3.4.1, item 1.
10763 addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName,
10764 OutputBecomesInput, Fn);
10765 } else {
10766 assert(ISA == 'n' && "Expected ISA either 's' or 'n'.")((ISA == 'n' && "Expected ISA either 's' or 'n'.") ? static_cast
<void> (0) : __assert_fail ("ISA == 'n' && \"Expected ISA either 's' or 'n'.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10766, __PRETTY_FUNCTION__))
;
10767 // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or
10768 // two vector names depending on the use of the clause
10769 // `[not]inbranch`.
10770 switch (State) {
10771 case OMPDeclareSimdDeclAttr::BS_Undefined:
10772 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
10773 OutputBecomesInput, Fn);
10774 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
10775 OutputBecomesInput, Fn);
10776 break;
10777 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10778 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
10779 OutputBecomesInput, Fn);
10780 break;
10781 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10782 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
10783 OutputBecomesInput, Fn);
10784 break;
10785 }
10786 }
10787 }
10788}
10789
10790void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
10791 llvm::Function *Fn) {
10792 ASTContext &C = CGM.getContext();
10793 FD = FD->getMostRecentDecl();
10794 // Map params to their positions in function decl.
10795 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10796 if (isa<CXXMethodDecl>(FD))
10797 ParamPositions.try_emplace(FD, 0);
10798 unsigned ParamPos = ParamPositions.size();
10799 for (const ParmVarDecl *P : FD->parameters()) {
10800 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
10801 ++ParamPos;
10802 }
10803 while (FD) {
10804 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
10805 llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
10806 // Mark uniform parameters.
10807 for (const Expr *E : Attr->uniforms()) {
10808 E = E->IgnoreParenImpCasts();
10809 unsigned Pos;
10810 if (isa<CXXThisExpr>(E)) {
10811 Pos = ParamPositions[FD];
10812 } else {
10813 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10814 ->getCanonicalDecl();
10815 Pos = ParamPositions[PVD];
10816 }
10817 ParamAttrs[Pos].Kind = Uniform;
10818 }
10819 // Get alignment info.
10820 auto NI = Attr->alignments_begin();
10821 for (const Expr *E : Attr->aligneds()) {
10822 E = E->IgnoreParenImpCasts();
10823 unsigned Pos;
10824 QualType ParmTy;
10825 if (isa<CXXThisExpr>(E)) {
10826 Pos = ParamPositions[FD];
10827 ParmTy = E->getType();
10828 } else {
10829 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10830 ->getCanonicalDecl();
10831 Pos = ParamPositions[PVD];
10832 ParmTy = PVD->getType();
10833 }
10834 ParamAttrs[Pos].Alignment =
10835 (*NI)
10836 ? (*NI)->EvaluateKnownConstInt(C)
10837 : llvm::APSInt::getUnsigned(
10838 C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
10839 .getQuantity());
10840 ++NI;
10841 }
10842 // Mark linear parameters.
10843 auto SI = Attr->steps_begin();
10844 auto MI = Attr->modifiers_begin();
10845 for (const Expr *E : Attr->linears()) {
10846 E = E->IgnoreParenImpCasts();
10847 unsigned Pos;
10848 if (isa<CXXThisExpr>(E)) {
10849 Pos = ParamPositions[FD];
10850 } else {
10851 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10852 ->getCanonicalDecl();
10853 Pos = ParamPositions[PVD];
10854 }
10855 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10856 ParamAttr.Kind = Linear;
10857 if (*SI) {
10858 Expr::EvalResult Result;
10859 if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {
10860 if (const auto *DRE =
10861 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
10862 if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
10863 ParamAttr.Kind = LinearWithVarStride;
10864 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
10865 ParamPositions[StridePVD->getCanonicalDecl()]);
10866 }
10867 }
10868 } else {
10869 ParamAttr.StrideOrArg = Result.Val.getInt();
10870 }
10871 }
10872 ++SI;
10873 ++MI;
10874 }
10875 llvm::APSInt VLENVal;
10876 SourceLocation ExprLoc;
10877 const Expr *VLENExpr = Attr->getSimdlen();
10878 if (VLENExpr) {
10879 VLENVal = VLENExpr->EvaluateKnownConstInt(C);
10880 ExprLoc = VLENExpr->getExprLoc();
10881 }
10882 OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
10883 if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
10884 CGM.getTriple().getArch() == llvm::Triple::x86_64) {
10885 emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
10886 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
10887 unsigned VLEN = VLENVal.getExtValue();
10888 StringRef MangledName = Fn->getName();
10889 if (CGM.getTarget().hasFeature("sve"))
10890 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
10891 MangledName, 's', 128, Fn, ExprLoc);
10892 if (CGM.getTarget().hasFeature("neon"))
10893 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
10894 MangledName, 'n', 128, Fn, ExprLoc);
10895 }
10896 }
10897 FD = FD->getPreviousDecl();
10898 }
10899}
10900
10901namespace {
10902/// Cleanup action for doacross support.
10903class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
10904public:
10905 static const int DoacrossFinArgs = 2;
10906
10907private:
10908 llvm::FunctionCallee RTLFn;
10909 llvm::Value *Args[DoacrossFinArgs];
10910
10911public:
10912 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
10913 ArrayRef<llvm::Value *> CallArgs)
10914 : RTLFn(RTLFn) {
10915 assert(CallArgs.size() == DoacrossFinArgs)((CallArgs.size() == DoacrossFinArgs) ? static_cast<void>
(0) : __assert_fail ("CallArgs.size() == DoacrossFinArgs", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 10915, __PRETTY_FUNCTION__))
;
10916 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10917 }
10918 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
10919 if (!CGF.HaveInsertPoint())
10920 return;
10921 CGF.EmitRuntimeCall(RTLFn, Args);
10922 }
10923};
10924} // namespace
10925
10926void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
10927 const OMPLoopDirective &D,
10928 ArrayRef<Expr *> NumIterations) {
10929 if (!CGF.HaveInsertPoint())
10930 return;
10931
10932 ASTContext &C = CGM.getContext();
10933 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
10934 RecordDecl *RD;
10935 if (KmpDimTy.isNull()) {
10936 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
10937 // kmp_int64 lo; // lower
10938 // kmp_int64 up; // upper
10939 // kmp_int64 st; // stride
10940 // };
10941 RD = C.buildImplicitRecord("kmp_dim");
10942 RD->startDefinition();
10943 addFieldToRecordDecl(C, RD, Int64Ty);
10944 addFieldToRecordDecl(C, RD, Int64Ty);
10945 addFieldToRecordDecl(C, RD, Int64Ty);
10946 RD->completeDefinition();
10947 KmpDimTy = C.getRecordType(RD);
10948 } else {
10949 RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
10950 }
10951 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
10952 QualType ArrayTy =
10953 C.getConstantArrayType(KmpDimTy, Size, ArrayType::Normal, 0);
10954
10955 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
10956 CGF.EmitNullInitialization(DimsAddr, ArrayTy);
10957 enum { LowerFD = 0, UpperFD, StrideFD };
10958 // Fill dims with data.
10959 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
10960 LValue DimsLVal = CGF.MakeAddrLValue(
10961 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
10962 // dims.upper = num_iterations;
10963 LValue UpperLVal = CGF.EmitLValueForField(
10964 DimsLVal, *std::next(RD->field_begin(), UpperFD));
10965 llvm::Value *NumIterVal =
10966 CGF.EmitScalarConversion(CGF.EmitScalarExpr(NumIterations[I]),
10967 D.getNumIterations()->getType(), Int64Ty,
10968 D.getNumIterations()->getExprLoc());
10969 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
10970 // dims.stride = 1;
10971 LValue StrideLVal = CGF.EmitLValueForField(
10972 DimsLVal, *std::next(RD->field_begin(), StrideFD));
10973 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
10974 StrideLVal);
10975 }
10976
10977 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
10978 // kmp_int32 num_dims, struct kmp_dim * dims);
10979 llvm::Value *Args[] = {
10980 emitUpdateLocation(CGF, D.getBeginLoc()),
10981 getThreadID(CGF, D.getBeginLoc()),
10982 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
10983 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
10984 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
10985 CGM.VoidPtrTy)};
10986
10987 llvm::FunctionCallee RTLFn =
10988 createRuntimeFunction(OMPRTL__kmpc_doacross_init);
10989 CGF.EmitRuntimeCall(RTLFn, Args);
10990 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
10991 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
10992 llvm::FunctionCallee FiniRTLFn =
10993 createRuntimeFunction(OMPRTL__kmpc_doacross_fini);
10994 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
10995 llvm::makeArrayRef(FiniArgs));
10996}
10997
10998void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
10999 const OMPDependClause *C) {
11000 QualType Int64Ty =
11001 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
11002 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
11003 QualType ArrayTy = CGM.getContext().getConstantArrayType(
11004 Int64Ty, Size, ArrayType::Normal, 0);
11005 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
11006 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
11007 const Expr *CounterVal = C->getLoopData(I);
11008 assert(CounterVal)((CounterVal) ? static_cast<void> (0) : __assert_fail (
"CounterVal", "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11008, __PRETTY_FUNCTION__))
;
11009 llvm::Value *CntVal = CGF.EmitScalarConversion(
11010 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
11011 CounterVal->getExprLoc());
11012 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
11013 /*Volatile=*/false, Int64Ty);
11014 }
11015 llvm::Value *Args[] = {
11016 emitUpdateLocation(CGF, C->getBeginLoc()),
11017 getThreadID(CGF, C->getBeginLoc()),
11018 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
11019 llvm::FunctionCallee RTLFn;
11020 if (C->getDependencyKind() == OMPC_DEPEND_source) {
11021 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post);
11022 } else {
11023 assert(C->getDependencyKind() == OMPC_DEPEND_sink)((C->getDependencyKind() == OMPC_DEPEND_sink) ? static_cast
<void> (0) : __assert_fail ("C->getDependencyKind() == OMPC_DEPEND_sink"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11023, __PRETTY_FUNCTION__))
;
11024 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait);
11025 }
11026 CGF.EmitRuntimeCall(RTLFn, Args);
11027}
11028
11029void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
11030 llvm::FunctionCallee Callee,
11031 ArrayRef<llvm::Value *> Args) const {
11032 assert(Loc.isValid() && "Outlined function call location must be valid.")((Loc.isValid() && "Outlined function call location must be valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Outlined function call location must be valid.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11032, __PRETTY_FUNCTION__))
;
11033 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
11034
11035 if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11036 if (Fn->doesNotThrow()) {
11037 CGF.EmitNounwindRuntimeCall(Fn, Args);
11038 return;
11039 }
11040 }
11041 CGF.EmitRuntimeCall(Callee, Args);
11042}
11043
11044void CGOpenMPRuntime::emitOutlinedFunctionCall(
11045 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
11046 ArrayRef<llvm::Value *> Args) const {
11047 emitCall(CGF, Loc, OutlinedFn, Args);
11048}
11049
11050void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
11051 if (const auto *FD = dyn_cast<FunctionDecl>(D))
11052 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11053 HasEmittedDeclareTargetRegion = true;
11054}
11055
11056Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
11057 const VarDecl *NativeParam,
11058 const VarDecl *TargetParam) const {
11059 return CGF.GetAddrOfLocalVar(NativeParam);
11060}
11061
11062namespace {
11063/// Cleanup action for allocate support.
11064class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
11065public:
11066 static const int CleanupArgs = 3;
11067
11068private:
11069 llvm::FunctionCallee RTLFn;
11070 llvm::Value *Args[CleanupArgs];
11071
11072public:
11073 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11074 ArrayRef<llvm::Value *> CallArgs)
11075 : RTLFn(RTLFn) {
11076 assert(CallArgs.size() == CleanupArgs &&((CallArgs.size() == CleanupArgs && "Size of arguments does not match."
) ? static_cast<void> (0) : __assert_fail ("CallArgs.size() == CleanupArgs && \"Size of arguments does not match.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11077, __PRETTY_FUNCTION__))
11077 "Size of arguments does not match.")((CallArgs.size() == CleanupArgs && "Size of arguments does not match."
) ? static_cast<void> (0) : __assert_fail ("CallArgs.size() == CleanupArgs && \"Size of arguments does not match.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11077, __PRETTY_FUNCTION__))
;
11078 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11079 }
11080 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
11081 if (!CGF.HaveInsertPoint())
11082 return;
11083 CGF.EmitRuntimeCall(RTLFn, Args);
11084 }
11085};
11086} // namespace
11087
11088Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
11089 const VarDecl *VD) {
11090 if (!VD)
11091 return Address::invalid();
11092 const VarDecl *CVD = VD->getCanonicalDecl();
11093 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
11094 return Address::invalid();
11095 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
11096 // Use the default allocation.
11097 if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
11098 !AA->getAllocator())
11099 return Address::invalid();
11100 llvm::Value *Size;
11101 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
11102 if (CVD->getType()->isVariablyModifiedType()) {
11103 Size = CGF.getTypeSize(CVD->getType());
11104 // Align the size: ((size + align - 1) / align) * align
11105 Size = CGF.Builder.CreateNUWAdd(
11106 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
11107 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
11108 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
11109 } else {
11110 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
11111 Size = CGM.getSize(Sz.alignTo(Align));
11112 }
11113 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
11114 assert(AA->getAllocator() &&((AA->getAllocator() && "Expected allocator expression for non-default allocator."
) ? static_cast<void> (0) : __assert_fail ("AA->getAllocator() && \"Expected allocator expression for non-default allocator.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11115, __PRETTY_FUNCTION__))
11115 "Expected allocator expression for non-default allocator.")((AA->getAllocator() && "Expected allocator expression for non-default allocator."
) ? static_cast<void> (0) : __assert_fail ("AA->getAllocator() && \"Expected allocator expression for non-default allocator.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11115, __PRETTY_FUNCTION__))
;
11116 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
11117 // According to the standard, the original allocator type is a enum (integer).
11118 // Convert to pointer type, if required.
11119 if (Allocator->getType()->isIntegerTy())
11120 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
11121 else if (Allocator->getType()->isPointerTy())
11122 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
11123 CGM.VoidPtrTy);
11124 llvm::Value *Args[] = {ThreadID, Size, Allocator};
11125
11126 llvm::Value *Addr =
11127 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_alloc), Args,
11128 CVD->getName() + ".void.addr");
11129 llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
11130 Allocator};
11131 llvm::FunctionCallee FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_free);
11132
11133 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
11134 llvm::makeArrayRef(FiniArgs));
11135 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11136 Addr,
11137 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
11138 CVD->getName() + ".addr");
11139 return Address(Addr, Align);
11140}
11141
11142/// Checks current context and returns true if it matches the context selector.
11143template <OMPDeclareVariantAttr::CtxSelectorSetType CtxSet,
11144 OMPDeclareVariantAttr::CtxSelectorType Ctx>
11145static bool checkContext(const OMPDeclareVariantAttr *A) {
11146 assert(CtxSet != OMPDeclareVariantAttr::CtxSetUnknown &&((CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx
!= OMPDeclareVariantAttr::CtxUnknown && "Unknown context selector or context selector set."
) ? static_cast<void> (0) : __assert_fail ("CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx != OMPDeclareVariantAttr::CtxUnknown && \"Unknown context selector or context selector set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11148, __PRETTY_FUNCTION__))
11147 Ctx != OMPDeclareVariantAttr::CtxUnknown &&((CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx
!= OMPDeclareVariantAttr::CtxUnknown && "Unknown context selector or context selector set."
) ? static_cast<void> (0) : __assert_fail ("CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx != OMPDeclareVariantAttr::CtxUnknown && \"Unknown context selector or context selector set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11148, __PRETTY_FUNCTION__))
11148 "Unknown context selector or context selector set.")((CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx
!= OMPDeclareVariantAttr::CtxUnknown && "Unknown context selector or context selector set."
) ? static_cast<void> (0) : __assert_fail ("CtxSet != OMPDeclareVariantAttr::CtxSetUnknown && Ctx != OMPDeclareVariantAttr::CtxUnknown && \"Unknown context selector or context selector set.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11148, __PRETTY_FUNCTION__))
;
11149 return false;
11150}
11151
11152/// Checks for implementation={vendor(<vendor>)} context selector.
11153/// \returns true iff <vendor>="llvm", false otherwise.
11154template <>
11155bool checkContext<OMPDeclareVariantAttr::CtxSetImplementation,
11156 OMPDeclareVariantAttr::CtxVendor>(
11157 const OMPDeclareVariantAttr *A) {
11158 return !A->getImplVendor().compare("llvm");
11159}
11160
11161/// Finds the variant function that matches current context with its context
11162/// selector.
11163static const FunctionDecl *getDeclareVariantFunction(const FunctionDecl *FD) {
11164 if (!FD->hasAttrs() || !FD->hasAttr<OMPDeclareVariantAttr>())
11165 return FD;
11166 // Iterate through all DeclareVariant attributes and check context selectors.
11167 SmallVector<const OMPDeclareVariantAttr *, 4> MatchingAttributes;
11168 for (const auto * A : FD->specific_attrs<OMPDeclareVariantAttr>()) {
11169 switch (A->getCtxSelectorSet()) {
11170 case OMPDeclareVariantAttr::CtxSetImplementation:
11171 switch (A->getCtxSelector()) {
11172 case OMPDeclareVariantAttr::CtxVendor:
11173 if (checkContext<OMPDeclareVariantAttr::CtxSetImplementation,
11174 OMPDeclareVariantAttr::CtxVendor>(A))
11175 MatchingAttributes.push_back(A);
11176 break;
11177 case OMPDeclareVariantAttr::CtxUnknown:
11178 llvm_unreachable(::llvm::llvm_unreachable_internal("Unknown context selector in implementation selctor set."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11179)
11179 "Unknown context selector in implementation selctor set.")::llvm::llvm_unreachable_internal("Unknown context selector in implementation selctor set."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11179)
;
11180 }
11181 break;
11182 case OMPDeclareVariantAttr::CtxSetUnknown:
11183 llvm_unreachable("Unknown context selector set.")::llvm::llvm_unreachable_internal("Unknown context selector set."
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11183)
;
11184 }
11185 }
11186 if (MatchingAttributes.empty())
11187 return FD;
11188 // TODO: implement score analysis of multiple context selectors.
11189 const OMPDeclareVariantAttr *MainAttr = MatchingAttributes.front();
11190 return cast<FunctionDecl>(
11191 cast<DeclRefExpr>(MainAttr->getVariantFuncRef()->IgnoreParenImpCasts())
11192 ->getDecl());
11193}
11194
11195bool CGOpenMPRuntime::emitDeclareVariant(GlobalDecl GD, bool IsForDefinition) {
11196 const auto *D = cast<FunctionDecl>(GD.getDecl());
11197 // If the original function is defined already, use its definition.
11198 StringRef MangledName = CGM.getMangledName(GD);
11199 llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName);
11200 if (Orig && !Orig->isDeclaration())
11201 return false;
11202 const FunctionDecl *NewFD = getDeclareVariantFunction(D);
11203 // Emit original function if it does not have declare variant attribute or the
11204 // context does not match.
11205 if (NewFD == D)
11206 return false;
11207 GlobalDecl NewGD = GD.getWithDecl(NewFD);
11208 if (tryEmitAlias(CGM, NewGD, GD, Orig, IsForDefinition)) {
11209 DeferredVariantFunction.erase(D);
11210 return true;
11211 }
11212 DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD)));
11213 return true;
11214}
11215
11216llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
11217 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11218 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
11219 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11219)
;
11220}
11221
11222llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
11223 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11224 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
11225 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11225)
;
11226}
11227
11228llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
11229 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11230 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
11231 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
11232 bool Tied, unsigned &NumberOfParts) {
11233 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11233)
;
11234}
11235
11236void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
11237 SourceLocation Loc,
11238 llvm::Function *OutlinedFn,
11239 ArrayRef<llvm::Value *> CapturedVars,
11240 const Expr *IfCond) {
11241 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11241)
;
11242}
11243
11244void CGOpenMPSIMDRuntime::emitCriticalRegion(
11245 CodeGenFunction &CGF, StringRef CriticalName,
11246 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
11247 const Expr *Hint) {
11248 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11248)
;
11249}
11250
11251void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
11252 const RegionCodeGenTy &MasterOpGen,
11253 SourceLocation Loc) {
11254 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11254)
;
11255}
11256
11257void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
11258 SourceLocation Loc) {
11259 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11259)
;
11260}
11261
11262void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
11263 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
11264 SourceLocation Loc) {
11265 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11265)
;
11266}
11267
11268void CGOpenMPSIMDRuntime::emitSingleRegion(
11269 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
11270 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
11271 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
11272 ArrayRef<const Expr *> AssignmentOps) {
11273 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11273)
;
11274}
11275
11276void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
11277 const RegionCodeGenTy &OrderedOpGen,
11278 SourceLocation Loc,
11279 bool IsThreads) {
11280 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11280)
;
11281}
11282
11283void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
11284 SourceLocation Loc,
11285 OpenMPDirectiveKind Kind,
11286 bool EmitChecks,
11287 bool ForceSimpleCall) {
11288 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11288)
;
11289}
11290
11291void CGOpenMPSIMDRuntime::emitForDispatchInit(
11292 CodeGenFunction &CGF, SourceLocation Loc,
11293 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
11294 bool Ordered, const DispatchRTInput &DispatchValues) {
11295 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11295)
;
11296}
11297
11298void CGOpenMPSIMDRuntime::emitForStaticInit(
11299 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
11300 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
11301 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11301)
;
11302}
11303
11304void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
11305 CodeGenFunction &CGF, SourceLocation Loc,
11306 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
11307 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11307)
;
11308}
11309
11310void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
11311 SourceLocation Loc,
11312 unsigned IVSize,
11313 bool IVSigned) {
11314 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11314)
;
11315}
11316
11317void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
11318 SourceLocation Loc,
11319 OpenMPDirectiveKind DKind) {
11320 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11320)
;
11321}
11322
11323llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
11324 SourceLocation Loc,
11325 unsigned IVSize, bool IVSigned,
11326 Address IL, Address LB,
11327 Address UB, Address ST) {
11328 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11328)
;
11329}
11330
11331void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
11332 llvm::Value *NumThreads,
11333 SourceLocation Loc) {
11334 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11334)
;
11335}
11336
11337void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
11338 OpenMPProcBindClauseKind ProcBind,
11339 SourceLocation Loc) {
11340 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11340)
;
11341}
11342
11343Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
11344 const VarDecl *VD,
11345 Address VDAddr,
11346 SourceLocation Loc) {
11347 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11347)
;
11348}
11349
11350llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
11351 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
11352 CodeGenFunction *CGF) {
11353 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11353)
;
11354}
11355
11356Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
11357 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
11358 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11358)
;
11359}
11360
11361void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
11362 ArrayRef<const Expr *> Vars,
11363 SourceLocation Loc) {
11364 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11364)
;
11365}
11366
11367void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
11368 const OMPExecutableDirective &D,
11369 llvm::Function *TaskFunction,
11370 QualType SharedsTy, Address Shareds,
11371 const Expr *IfCond,
11372 const OMPTaskDataTy &Data) {
11373 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11373)
;
11374}
11375
11376void CGOpenMPSIMDRuntime::emitTaskLoopCall(
11377 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
11378 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
11379 const Expr *IfCond, const OMPTaskDataTy &Data) {
11380 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11380)
;
11381}
11382
11383void CGOpenMPSIMDRuntime::emitReduction(
11384 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
11385 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
11386 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
11387 assert(Options.SimpleReduction && "Only simple reduction is expected.")((Options.SimpleReduction && "Only simple reduction is expected."
) ? static_cast<void> (0) : __assert_fail ("Options.SimpleReduction && \"Only simple reduction is expected.\""
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11387, __PRETTY_FUNCTION__))
;
11388 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
11389 ReductionOps, Options);
11390}
11391
11392llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
11393 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
11394 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
11395 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11395)
;
11396}
11397
11398void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
11399 SourceLocation Loc,
11400 ReductionCodeGen &RCG,
11401 unsigned N) {
11402 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11402)
;
11403}
11404
11405Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
11406 SourceLocation Loc,
11407 llvm::Value *ReductionsPtr,
11408 LValue SharedLVal) {
11409 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11409)
;
11410}
11411
11412void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
11413 SourceLocation Loc) {
11414 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11414)
;
11415}
11416
11417void CGOpenMPSIMDRuntime::emitCancellationPointCall(
11418 CodeGenFunction &CGF, SourceLocation Loc,
11419 OpenMPDirectiveKind CancelRegion) {
11420 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11420)
;
11421}
11422
11423void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
11424 SourceLocation Loc, const Expr *IfCond,
11425 OpenMPDirectiveKind CancelRegion) {
11426 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11426)
;
11427}
11428
11429void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
11430 const OMPExecutableDirective &D, StringRef ParentName,
11431 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
11432 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
11433 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11433)
;
11434}
11435
11436void CGOpenMPSIMDRuntime::emitTargetCall(
11437 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11438 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11439 const Expr *Device,
11440 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11441 const OMPLoopDirective &D)>
11442 SizeEmitter) {
11443 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11443)
;
11444}
11445
11446bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
11447 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11447)
;
11448}
11449
11450bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11451 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11451)
;
11452}
11453
11454bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
11455 return false;
11456}
11457
11458llvm::Function *CGOpenMPSIMDRuntime::emitRegistrationFunction() {
11459 return nullptr;
11460}
11461
11462void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
11463 const OMPExecutableDirective &D,
11464 SourceLocation Loc,
11465 llvm::Function *OutlinedFn,
11466 ArrayRef<llvm::Value *> CapturedVars) {
11467 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11467)
;
11468}
11469
11470void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11471 const Expr *NumTeams,
11472 const Expr *ThreadLimit,
11473 SourceLocation Loc) {
11474 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11474)
;
11475}
11476
11477void CGOpenMPSIMDRuntime::emitTargetDataCalls(
11478 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11479 const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
11480 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11480)
;
11481}
11482
11483void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
11484 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11485 const Expr *Device) {
11486 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11486)
;
11487}
11488
11489void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
11490 const OMPLoopDirective &D,
11491 ArrayRef<Expr *> NumIterations) {
11492 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11492)
;
11493}
11494
11495void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11496 const OMPDependClause *C) {
11497 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11497)
;
11498}
11499
11500const VarDecl *
11501CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
11502 const VarDecl *NativeParam) const {
11503 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11503)
;
11504}
11505
11506Address
11507CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
11508 const VarDecl *NativeParam,
11509 const VarDecl *TargetParam) const {
11510 llvm_unreachable("Not supported in SIMD-only mode")::llvm::llvm_unreachable_internal("Not supported in SIMD-only mode"
, "/build/llvm-toolchain-snapshot-10~svn373517/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 11510)
;
11511}

/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h

1//===- Optional.h - Simple variant for passing optional values --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides Optional, a template class modeled in the spirit of
10// OCaml's 'opt' variant. The idea is to strongly type whether or not
11// a value can be optional.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_ADT_OPTIONAL_H
16#define LLVM_ADT_OPTIONAL_H
17
18#include "llvm/ADT/None.h"
19#include "llvm/Support/Compiler.h"
20#include "llvm/Support/type_traits.h"
21#include <cassert>
22#include <memory>
23#include <new>
24#include <utility>
25
26namespace llvm {
27
28class raw_ostream;
29
30namespace optional_detail {
31
32struct in_place_t {};
33
34/// Storage for any type.
35template <typename T, bool = is_trivially_copyable<T>::value>
36class OptionalStorage {
37 union {
38 char empty;
39 T value;
40 };
41 bool hasVal;
42
43public:
44 ~OptionalStorage() { reset(); }
45
46 OptionalStorage() noexcept : empty(), hasVal(false) {}
47
48 OptionalStorage(OptionalStorage const &other) : OptionalStorage() {
49 if (other.hasValue()) {
50 emplace(other.value);
51 }
52 }
53 OptionalStorage(OptionalStorage &&other) : OptionalStorage() {
54 if (other.hasValue()) {
55 emplace(std::move(other.value));
56 }
57 }
58
59 template <class... Args>
60 explicit OptionalStorage(in_place_t, Args &&... args)
61 : value(std::forward<Args>(args)...), hasVal(true) {}
62
63 void reset() noexcept {
64 if (hasVal) {
65 value.~T();
66 hasVal = false;
67 }
68 }
69
70 bool hasValue() const noexcept { return hasVal; }
71
72 T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
73 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 73, __PRETTY_FUNCTION__))
;
74 return value;
75 }
76 T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
77 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 77, __PRETTY_FUNCTION__))
;
78 return value;
79 }
80#if LLVM_HAS_RVALUE_REFERENCE_THIS1
81 T &&getValue() && noexcept {
82 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 82, __PRETTY_FUNCTION__))
;
83 return std::move(value);
84 }
85#endif
86
87 template <class... Args> void emplace(Args &&... args) {
88 reset();
89 ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
90 hasVal = true;
91 }
92
93 OptionalStorage &operator=(T const &y) {
94 if (hasValue()) {
95 value = y;
96 } else {
97 ::new ((void *)std::addressof(value)) T(y);
98 hasVal = true;
99 }
100 return *this;
101 }
102 OptionalStorage &operator=(T &&y) {
103 if (hasValue()) {
104 value = std::move(y);
105 } else {
106 ::new ((void *)std::addressof(value)) T(std::move(y));
107 hasVal = true;
108 }
109 return *this;
110 }
111
112 OptionalStorage &operator=(OptionalStorage const &other) {
113 if (other.hasValue()) {
114 if (hasValue()) {
115 value = other.value;
116 } else {
117 ::new ((void *)std::addressof(value)) T(other.value);
118 hasVal = true;
119 }
120 } else {
121 reset();
122 }
123 return *this;
124 }
125
126 OptionalStorage &operator=(OptionalStorage &&other) {
127 if (other.hasValue()) {
128 if (hasValue()) {
129 value = std::move(other.value);
130 } else {
131 ::new ((void *)std::addressof(value)) T(std::move(other.value));
132 hasVal = true;
133 }
134 } else {
135 reset();
136 }
137 return *this;
138 }
139};
140
141template <typename T> class OptionalStorage<T, true> {
142 union {
143 char empty;
144 T value;
145 };
146 bool hasVal = false;
147
148public:
149 ~OptionalStorage() = default;
150
151 OptionalStorage() noexcept : empty{} {}
152
153 OptionalStorage(OptionalStorage const &other) = default;
154 OptionalStorage(OptionalStorage &&other) = default;
155
156 OptionalStorage &operator=(OptionalStorage const &other) = default;
157 OptionalStorage &operator=(OptionalStorage &&other) = default;
158
159 template <class... Args>
160 explicit OptionalStorage(in_place_t, Args &&... args)
161 : value(std::forward<Args>(args)...), hasVal(true) {}
162
163 void reset() noexcept {
164 if (hasVal) {
165 value.~T();
166 hasVal = false;
167 }
168 }
169
170 bool hasValue() const noexcept { return hasVal; }
6
Returning the value 1, which participates in a condition later
171
172 T &getValue() LLVM_LVALUE_FUNCTION& noexcept {
173 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 173, __PRETTY_FUNCTION__))
;
174 return value;
175 }
176 T const &getValue() const LLVM_LVALUE_FUNCTION& noexcept {
177 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 177, __PRETTY_FUNCTION__))
;
178 return value;
179 }
180#if LLVM_HAS_RVALUE_REFERENCE_THIS1
181 T &&getValue() && noexcept {
182 assert(hasVal)((hasVal) ? static_cast<void> (0) : __assert_fail ("hasVal"
, "/build/llvm-toolchain-snapshot-10~svn373517/include/llvm/ADT/Optional.h"
, 182, __PRETTY_FUNCTION__))
;
183 return std::move(value);
184 }
185#endif
186
187 template <class... Args> void emplace(Args &&... args) {
188 reset();
189 ::new ((void *)std::addressof(value)) T(std::forward<Args>(args)...);
190 hasVal = true;
191 }
192
193 OptionalStorage &operator=(T const &y) {
194 if (hasValue()) {
195 value = y;
196 } else {
197 ::new ((void *)std::addressof(value)) T(y);
198 hasVal = true;
199 }
200 return *this;
201 }
202 OptionalStorage &operator=(T &&y) {
203 if (hasValue()) {
204 value = std::move(y);
205 } else {
206 ::new ((void *)std::addressof(value)) T(std::move(y));
207 hasVal = true;
208 }
209 return *this;
210 }
211};
212
213} // namespace optional_detail
214
215template <typename T> class Optional {
216 optional_detail::OptionalStorage<T> Storage;
217
218public:
219 using value_type = T;
220
221 constexpr Optional() {}
222 constexpr Optional(NoneType) {}
223
224 Optional(const T &y) : Storage(optional_detail::in_place_t{}, y) {}
225 Optional(const Optional &O) = default;
226
227 Optional(T &&y) : Storage(optional_detail::in_place_t{}, std::move(y)) {}
228 Optional(Optional &&O) = default;
229
230 Optional &operator=(T &&y) {
231 Storage = std::move(y);
232 return *this;
233 }
234 Optional &operator=(Optional &&O) = default;
235
236 /// Create a new object by constructing it in place with the given arguments.
237 template <typename... ArgTypes> void emplace(ArgTypes &&... Args) {
238 Storage.emplace(std::forward<ArgTypes>(Args)...);
239 }
240
241 static inline Optional create(const T *y) {
242 return y ? Optional(*y) : Optional();
243 }
244
245 Optional &operator=(const T &y) {
246 Storage = y;
247 return *this;
248 }
249 Optional &operator=(const Optional &O) = default;
250
251 void reset() { Storage.reset(); }
252
253 const T *getPointer() const { return &Storage.getValue(); }
254 T *getPointer() { return &Storage.getValue(); }
255 const T &getValue() const LLVM_LVALUE_FUNCTION& { return Storage.getValue(); }
256 T &getValue() LLVM_LVALUE_FUNCTION& { return Storage.getValue(); }
257
258 explicit operator bool() const { return hasValue(); }
4
Calling 'Optional::hasValue'
9
Returning from 'Optional::hasValue'
10
Returning the value 1, which participates in a condition later
259 bool hasValue() const { return Storage.hasValue(); }
5
Calling 'OptionalStorage::hasValue'
7
Returning from 'OptionalStorage::hasValue'
8
Returning the value 1, which participates in a condition later
260 const T *operator->() const { return getPointer(); }
261 T *operator->() { return getPointer(); }
262 const T &operator*() const LLVM_LVALUE_FUNCTION& { return getValue(); }
263 T &operator*() LLVM_LVALUE_FUNCTION& { return getValue(); }
264
265 template <typename U>
266 constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION& {
267 return hasValue() ? getValue() : std::forward<U>(value);
268 }
269
270#if LLVM_HAS_RVALUE_REFERENCE_THIS1
271 T &&getValue() && { return std::move(Storage.getValue()); }
272 T &&operator*() && { return std::move(Storage.getValue()); }
273
274 template <typename U>
275 T getValueOr(U &&value) && {
276 return hasValue() ? std::move(getValue()) : std::forward<U>(value);
277 }
278#endif
279};
280
281template <typename T, typename U>
282bool operator==(const Optional<T> &X, const Optional<U> &Y) {
283 if (X && Y)
284 return *X == *Y;
285 return X.hasValue() == Y.hasValue();
286}
287
288template <typename T, typename U>
289bool operator!=(const Optional<T> &X, const Optional<U> &Y) {
290 return !(X == Y);
291}
292
293template <typename T, typename U>
294bool operator<(const Optional<T> &X, const Optional<U> &Y) {
295 if (X && Y)
296 return *X < *Y;
297 return X.hasValue() < Y.hasValue();
298}
299
300template <typename T, typename U>
301bool operator<=(const Optional<T> &X, const Optional<U> &Y) {
302 return !(Y < X);
303}
304
305template <typename T, typename U>
306bool operator>(const Optional<T> &X, const Optional<U> &Y) {
307 return Y < X;
308}
309
310template <typename T, typename U>
311bool operator>=(const Optional<T> &X, const Optional<U> &Y) {
312 return !(X < Y);
313}
314
315template<typename T>
316bool operator==(const Optional<T> &X, NoneType) {
317 return !X;
318}
319
320template<typename T>
321bool operator==(NoneType, const Optional<T> &X) {
322 return X == None;
323}
324
325template<typename T>
326bool operator!=(const Optional<T> &X, NoneType) {
327 return !(X == None);
328}
329
330template<typename T>
331bool operator!=(NoneType, const Optional<T> &X) {
332 return X != None;
333}
334
335template <typename T> bool operator<(const Optional<T> &X, NoneType) {
336 return false;
337}
338
339template <typename T> bool operator<(NoneType, const Optional<T> &X) {
340 return X.hasValue();
341}
342
343template <typename T> bool operator<=(const Optional<T> &X, NoneType) {
344 return !(None < X);
345}
346
347template <typename T> bool operator<=(NoneType, const Optional<T> &X) {
348 return !(X < None);
349}
350
351template <typename T> bool operator>(const Optional<T> &X, NoneType) {
352 return None < X;
353}
354
355template <typename T> bool operator>(NoneType, const Optional<T> &X) {
356 return X < None;
357}
358
359template <typename T> bool operator>=(const Optional<T> &X, NoneType) {
360 return None <= X;
361}
362
363template <typename T> bool operator>=(NoneType, const Optional<T> &X) {
364 return X <= None;
365}
366
367template <typename T> bool operator==(const Optional<T> &X, const T &Y) {
368 return X && *X == Y;
369}
370
371template <typename T> bool operator==(const T &X, const Optional<T> &Y) {
372 return Y && X == *Y;
373}
374
375template <typename T> bool operator!=(const Optional<T> &X, const T &Y) {
376 return !(X == Y);
377}
378
379template <typename T> bool operator!=(const T &X, const Optional<T> &Y) {
380 return !(X == Y);
381}
382
383template <typename T> bool operator<(const Optional<T> &X, const T &Y) {
384 return !X || *X < Y;
385}
386
387template <typename T> bool operator<(const T &X, const Optional<T> &Y) {
388 return Y && X < *Y;
389}
390
391template <typename T> bool operator<=(const Optional<T> &X, const T &Y) {
392 return !(Y < X);
393}
394
395template <typename T> bool operator<=(const T &X, const Optional<T> &Y) {
396 return !(Y < X);
397}
398
399template <typename T> bool operator>(const Optional<T> &X, const T &Y) {
400 return Y < X;
401}
402
403template <typename T> bool operator>(const T &X, const Optional<T> &Y) {
404 return Y < X;
405}
406
407template <typename T> bool operator>=(const Optional<T> &X, const T &Y) {
408 return !(X < Y);
409}
410
411template <typename T> bool operator>=(const T &X, const Optional<T> &Y) {
412 return !(X < Y);
413}
414
415raw_ostream &operator<<(raw_ostream &OS, NoneType);
416
417template <typename T, typename = decltype(std::declval<raw_ostream &>()
418 << std::declval<const T &>())>
419raw_ostream &operator<<(raw_ostream &OS, const Optional<T> &O) {
420 if (O)
421 OS << *O;
422 else
423 OS << None;
424 return OS;
425}
426
427} // end namespace llvm
428
429#endif // LLVM_ADT_OPTIONAL_H