Bug Summary

File:tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 7728, column 15
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-8~svn350071/tools/clang/include -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn350071/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-8~svn350071=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-27-042839-1215-1 -x c++ /build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This provides a class for OpenMP runtime code generation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGOpenMPRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "clang/CodeGen/ConstantInitBuilder.h"
20#include "clang/AST/Decl.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/Basic/BitmaskEnum.h"
23#include "llvm/ADT/ArrayRef.h"
24#include "llvm/Bitcode/BitcodeReader.h"
25#include "llvm/IR/CallSite.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/GlobalValue.h"
28#include "llvm/IR/Value.h"
29#include "llvm/Support/Format.h"
30#include "llvm/Support/raw_ostream.h"
31#include <cassert>
32
33using namespace clang;
34using namespace CodeGen;
35
36namespace {
37/// Base class for handling code generation inside OpenMP regions.
38class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
39public:
40 /// Kinds of OpenMP regions used in codegen.
41 enum CGOpenMPRegionKind {
42 /// Region with outlined function for standalone 'parallel'
43 /// directive.
44 ParallelOutlinedRegion,
45 /// Region with outlined function for standalone 'task' directive.
46 TaskOutlinedRegion,
47 /// Region for constructs that do not require function outlining,
48 /// like 'for', 'sections', 'atomic' etc. directives.
49 InlinedRegion,
50 /// Region with outlined function for standalone 'target' directive.
51 TargetRegion,
52 };
53
54 CGOpenMPRegionInfo(const CapturedStmt &CS,
55 const CGOpenMPRegionKind RegionKind,
56 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
57 bool HasCancel)
58 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
59 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
60
61 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
62 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
63 bool HasCancel)
64 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
65 Kind(Kind), HasCancel(HasCancel) {}
66
67 /// Get a variable or parameter for storing global thread id
68 /// inside OpenMP construct.
69 virtual const VarDecl *getThreadIDVariable() const = 0;
70
71 /// Emit the captured statement body.
72 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
73
74 /// Get an LValue for the current ThreadID variable.
75 /// \return LValue for thread id variable. This LValue always has type int32*.
76 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
77
78 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
79
80 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
81
82 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
83
84 bool hasCancel() const { return HasCancel; }
85
86 static bool classof(const CGCapturedStmtInfo *Info) {
87 return Info->getKind() == CR_OpenMP;
88 }
89
90 ~CGOpenMPRegionInfo() override = default;
91
92protected:
93 CGOpenMPRegionKind RegionKind;
94 RegionCodeGenTy CodeGen;
95 OpenMPDirectiveKind Kind;
96 bool HasCancel;
97};
98
99/// API for captured statement code generation in OpenMP constructs.
100class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
101public:
102 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
103 const RegionCodeGenTy &CodeGen,
104 OpenMPDirectiveKind Kind, bool HasCancel,
105 StringRef HelperName)
106 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
107 HasCancel),
108 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
109 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 109, __PRETTY_FUNCTION__))
;
110 }
111
112 /// Get a variable or parameter for storing global thread id
113 /// inside OpenMP construct.
114 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
115
116 /// Get the name of the capture helper.
117 StringRef getHelperName() const override { return HelperName; }
118
119 static bool classof(const CGCapturedStmtInfo *Info) {
120 return CGOpenMPRegionInfo::classof(Info) &&
121 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
122 ParallelOutlinedRegion;
123 }
124
125private:
126 /// A variable or parameter storing global thread id for OpenMP
127 /// constructs.
128 const VarDecl *ThreadIDVar;
129 StringRef HelperName;
130};
131
132/// API for captured statement code generation in OpenMP constructs.
133class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
134public:
135 class UntiedTaskActionTy final : public PrePostActionTy {
136 bool Untied;
137 const VarDecl *PartIDVar;
138 const RegionCodeGenTy UntiedCodeGen;
139 llvm::SwitchInst *UntiedSwitch = nullptr;
140
141 public:
142 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
143 const RegionCodeGenTy &UntiedCodeGen)
144 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
145 void Enter(CodeGenFunction &CGF) override {
146 if (Untied) {
147 // Emit task switching point.
148 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
149 CGF.GetAddrOfLocalVar(PartIDVar),
150 PartIDVar->getType()->castAs<PointerType>());
151 llvm::Value *Res =
152 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
153 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
154 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
155 CGF.EmitBlock(DoneBB);
156 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
157 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
158 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
159 CGF.Builder.GetInsertBlock());
160 emitUntiedSwitch(CGF);
161 }
162 }
163 void emitUntiedSwitch(CodeGenFunction &CGF) const {
164 if (Untied) {
165 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
166 CGF.GetAddrOfLocalVar(PartIDVar),
167 PartIDVar->getType()->castAs<PointerType>());
168 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
169 PartIdLVal);
170 UntiedCodeGen(CGF);
171 CodeGenFunction::JumpDest CurPoint =
172 CGF.getJumpDestInCurrentScope(".untied.next.");
173 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
174 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
175 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
176 CGF.Builder.GetInsertBlock());
177 CGF.EmitBranchThroughCleanup(CurPoint);
178 CGF.EmitBlock(CurPoint.getBlock());
179 }
180 }
181 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
182 };
183 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
184 const VarDecl *ThreadIDVar,
185 const RegionCodeGenTy &CodeGen,
186 OpenMPDirectiveKind Kind, bool HasCancel,
187 const UntiedTaskActionTy &Action)
188 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
189 ThreadIDVar(ThreadIDVar), Action(Action) {
190 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 190, __PRETTY_FUNCTION__))
;
191 }
192
193 /// Get a variable or parameter for storing global thread id
194 /// inside OpenMP construct.
195 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
196
197 /// Get an LValue for the current ThreadID variable.
198 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
199
200 /// Get the name of the capture helper.
201 StringRef getHelperName() const override { return ".omp_outlined."; }
202
203 void emitUntiedSwitch(CodeGenFunction &CGF) override {
204 Action.emitUntiedSwitch(CGF);
205 }
206
207 static bool classof(const CGCapturedStmtInfo *Info) {
208 return CGOpenMPRegionInfo::classof(Info) &&
209 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
210 TaskOutlinedRegion;
211 }
212
213private:
214 /// A variable or parameter storing global thread id for OpenMP
215 /// constructs.
216 const VarDecl *ThreadIDVar;
217 /// Action for emitting code for untied tasks.
218 const UntiedTaskActionTy &Action;
219};
220
221/// API for inlined captured statement code generation in OpenMP
222/// constructs.
223class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
224public:
225 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
226 const RegionCodeGenTy &CodeGen,
227 OpenMPDirectiveKind Kind, bool HasCancel)
228 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
229 OldCSI(OldCSI),
230 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
231
232 // Retrieve the value of the context parameter.
233 llvm::Value *getContextValue() const override {
234 if (OuterRegionInfo)
235 return OuterRegionInfo->getContextValue();
236 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 236)
;
237 }
238
239 void setContextValue(llvm::Value *V) override {
240 if (OuterRegionInfo) {
241 OuterRegionInfo->setContextValue(V);
242 return;
243 }
244 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 244)
;
245 }
246
247 /// Lookup the captured field decl for a variable.
248 const FieldDecl *lookup(const VarDecl *VD) const override {
249 if (OuterRegionInfo)
250 return OuterRegionInfo->lookup(VD);
251 // If there is no outer outlined region,no need to lookup in a list of
252 // captured variables, we can use the original one.
253 return nullptr;
254 }
255
256 FieldDecl *getThisFieldDecl() const override {
257 if (OuterRegionInfo)
258 return OuterRegionInfo->getThisFieldDecl();
259 return nullptr;
260 }
261
262 /// Get a variable or parameter for storing global thread id
263 /// inside OpenMP construct.
264 const VarDecl *getThreadIDVariable() const override {
265 if (OuterRegionInfo)
266 return OuterRegionInfo->getThreadIDVariable();
267 return nullptr;
268 }
269
270 /// Get an LValue for the current ThreadID variable.
271 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
272 if (OuterRegionInfo)
273 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
274 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 274)
;
275 }
276
277 /// Get the name of the capture helper.
278 StringRef getHelperName() const override {
279 if (auto *OuterRegionInfo = getOldCSI())
280 return OuterRegionInfo->getHelperName();
281 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 281)
;
282 }
283
284 void emitUntiedSwitch(CodeGenFunction &CGF) override {
285 if (OuterRegionInfo)
286 OuterRegionInfo->emitUntiedSwitch(CGF);
287 }
288
289 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
290
291 static bool classof(const CGCapturedStmtInfo *Info) {
292 return CGOpenMPRegionInfo::classof(Info) &&
293 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
294 }
295
296 ~CGOpenMPInlinedRegionInfo() override = default;
297
298private:
299 /// CodeGen info about outer OpenMP region.
300 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
301 CGOpenMPRegionInfo *OuterRegionInfo;
302};
303
304/// API for captured statement code generation in OpenMP target
305/// constructs. For this captures, implicit parameters are used instead of the
306/// captured fields. The name of the target region has to be unique in a given
307/// application so it is provided by the client, because only the client has
308/// the information to generate that.
309class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
310public:
311 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
312 const RegionCodeGenTy &CodeGen, StringRef HelperName)
313 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
314 /*HasCancel=*/false),
315 HelperName(HelperName) {}
316
317 /// This is unused for target regions because each starts executing
318 /// with a single thread.
319 const VarDecl *getThreadIDVariable() const override { return nullptr; }
320
321 /// Get the name of the capture helper.
322 StringRef getHelperName() const override { return HelperName; }
323
324 static bool classof(const CGCapturedStmtInfo *Info) {
325 return CGOpenMPRegionInfo::classof(Info) &&
326 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
327 }
328
329private:
330 StringRef HelperName;
331};
332
333static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
334 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 334)
;
335}
336/// API for generation of expressions captured in a innermost OpenMP
337/// region.
338class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
339public:
340 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
341 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
342 OMPD_unknown,
343 /*HasCancel=*/false),
344 PrivScope(CGF) {
345 // Make sure the globals captured in the provided statement are local by
346 // using the privatization logic. We assume the same variable is not
347 // captured more than once.
348 for (const auto &C : CS.captures()) {
349 if (!C.capturesVariable() && !C.capturesVariableByCopy())
350 continue;
351
352 const VarDecl *VD = C.getCapturedVar();
353 if (VD->isLocalVarDeclOrParm())
354 continue;
355
356 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
357 /*RefersToEnclosingVariableOrCapture=*/false,
358 VD->getType().getNonReferenceType(), VK_LValue,
359 C.getLocation());
360 PrivScope.addPrivate(
361 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
362 }
363 (void)PrivScope.Privatize();
364 }
365
366 /// Lookup the captured field decl for a variable.
367 const FieldDecl *lookup(const VarDecl *VD) const override {
368 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
369 return FD;
370 return nullptr;
371 }
372
373 /// Emit the captured statement body.
374 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
375 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 375)
;
376 }
377
378 /// Get a variable or parameter for storing global thread id
379 /// inside OpenMP construct.
380 const VarDecl *getThreadIDVariable() const override {
381 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 381)
;
382 }
383
384 /// Get the name of the capture helper.
385 StringRef getHelperName() const override {
386 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 386)
;
387 }
388
389 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
390
391private:
392 /// Private scope to capture global variables.
393 CodeGenFunction::OMPPrivateScope PrivScope;
394};
395
396/// RAII for emitting code of OpenMP constructs.
397class InlinedOpenMPRegionRAII {
398 CodeGenFunction &CGF;
399 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
400 FieldDecl *LambdaThisCaptureField = nullptr;
401 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
402
403public:
404 /// Constructs region for combined constructs.
405 /// \param CodeGen Code generation sequence for combined directives. Includes
406 /// a list of functions used for code generation of implicitly inlined
407 /// regions.
408 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
409 OpenMPDirectiveKind Kind, bool HasCancel)
410 : CGF(CGF) {
411 // Start emission for the construct.
412 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
413 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
414 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
415 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
416 CGF.LambdaThisCaptureField = nullptr;
417 BlockInfo = CGF.BlockInfo;
418 CGF.BlockInfo = nullptr;
419 }
420
421 ~InlinedOpenMPRegionRAII() {
422 // Restore original CapturedStmtInfo only if we're done with code emission.
423 auto *OldCSI =
424 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
425 delete CGF.CapturedStmtInfo;
426 CGF.CapturedStmtInfo = OldCSI;
427 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
428 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
429 CGF.BlockInfo = BlockInfo;
430 }
431};
432
433/// Values for bit flags used in the ident_t to describe the fields.
434/// All enumeric elements are named and described in accordance with the code
435/// from http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
436enum OpenMPLocationFlags : unsigned {
437 /// Use trampoline for internal microtask.
438 OMP_IDENT_IMD = 0x01,
439 /// Use c-style ident structure.
440 OMP_IDENT_KMPC = 0x02,
441 /// Atomic reduction option for kmpc_reduce.
442 OMP_ATOMIC_REDUCE = 0x10,
443 /// Explicit 'barrier' directive.
444 OMP_IDENT_BARRIER_EXPL = 0x20,
445 /// Implicit barrier in code.
446 OMP_IDENT_BARRIER_IMPL = 0x40,
447 /// Implicit barrier in 'for' directive.
448 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
449 /// Implicit barrier in 'sections' directive.
450 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
451 /// Implicit barrier in 'single' directive.
452 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
453 /// Call of __kmp_for_static_init for static loop.
454 OMP_IDENT_WORK_LOOP = 0x200,
455 /// Call of __kmp_for_static_init for sections.
456 OMP_IDENT_WORK_SECTIONS = 0x400,
457 /// Call of __kmp_for_static_init for distribute.
458 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
459 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
460};
461
462/// Describes ident structure that describes a source location.
463/// All descriptions are taken from
464/// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
465/// Original structure:
466/// typedef struct ident {
467/// kmp_int32 reserved_1; /**< might be used in Fortran;
468/// see above */
469/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
470/// KMP_IDENT_KMPC identifies this union
471/// member */
472/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
473/// see above */
474///#if USE_ITT_BUILD
475/// /* but currently used for storing
476/// region-specific ITT */
477/// /* contextual information. */
478///#endif /* USE_ITT_BUILD */
479/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
480/// C++ */
481/// char const *psource; /**< String describing the source location.
482/// The string is composed of semi-colon separated
483// fields which describe the source file,
484/// the function and a pair of line numbers that
485/// delimit the construct.
486/// */
487/// } ident_t;
488enum IdentFieldIndex {
489 /// might be used in Fortran
490 IdentField_Reserved_1,
491 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
492 IdentField_Flags,
493 /// Not really used in Fortran any more
494 IdentField_Reserved_2,
495 /// Source[4] in Fortran, do not use for C++
496 IdentField_Reserved_3,
497 /// String describing the source location. The string is composed of
498 /// semi-colon separated fields which describe the source file, the function
499 /// and a pair of line numbers that delimit the construct.
500 IdentField_PSource
501};
502
503/// Schedule types for 'omp for' loops (these enumerators are taken from
504/// the enum sched_type in kmp.h).
505enum OpenMPSchedType {
506 /// Lower bound for default (unordered) versions.
507 OMP_sch_lower = 32,
508 OMP_sch_static_chunked = 33,
509 OMP_sch_static = 34,
510 OMP_sch_dynamic_chunked = 35,
511 OMP_sch_guided_chunked = 36,
512 OMP_sch_runtime = 37,
513 OMP_sch_auto = 38,
514 /// static with chunk adjustment (e.g., simd)
515 OMP_sch_static_balanced_chunked = 45,
516 /// Lower bound for 'ordered' versions.
517 OMP_ord_lower = 64,
518 OMP_ord_static_chunked = 65,
519 OMP_ord_static = 66,
520 OMP_ord_dynamic_chunked = 67,
521 OMP_ord_guided_chunked = 68,
522 OMP_ord_runtime = 69,
523 OMP_ord_auto = 70,
524 OMP_sch_default = OMP_sch_static,
525 /// dist_schedule types
526 OMP_dist_sch_static_chunked = 91,
527 OMP_dist_sch_static = 92,
528 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
529 /// Set if the monotonic schedule modifier was present.
530 OMP_sch_modifier_monotonic = (1 << 29),
531 /// Set if the nonmonotonic schedule modifier was present.
532 OMP_sch_modifier_nonmonotonic = (1 << 30),
533};
534
535enum OpenMPRTLFunction {
536 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
537 /// kmpc_micro microtask, ...);
538 OMPRTL__kmpc_fork_call,
539 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
540 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
541 OMPRTL__kmpc_threadprivate_cached,
542 /// Call to void __kmpc_threadprivate_register( ident_t *,
543 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
544 OMPRTL__kmpc_threadprivate_register,
545 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
546 OMPRTL__kmpc_global_thread_num,
547 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
548 // kmp_critical_name *crit);
549 OMPRTL__kmpc_critical,
550 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
551 // global_tid, kmp_critical_name *crit, uintptr_t hint);
552 OMPRTL__kmpc_critical_with_hint,
553 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
554 // kmp_critical_name *crit);
555 OMPRTL__kmpc_end_critical,
556 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
557 // global_tid);
558 OMPRTL__kmpc_cancel_barrier,
559 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
560 OMPRTL__kmpc_barrier,
561 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
562 OMPRTL__kmpc_for_static_fini,
563 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
564 // global_tid);
565 OMPRTL__kmpc_serialized_parallel,
566 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
567 // global_tid);
568 OMPRTL__kmpc_end_serialized_parallel,
569 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
570 // kmp_int32 num_threads);
571 OMPRTL__kmpc_push_num_threads,
572 // Call to void __kmpc_flush(ident_t *loc);
573 OMPRTL__kmpc_flush,
574 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
575 OMPRTL__kmpc_master,
576 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
577 OMPRTL__kmpc_end_master,
578 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
579 // int end_part);
580 OMPRTL__kmpc_omp_taskyield,
581 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
582 OMPRTL__kmpc_single,
583 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
584 OMPRTL__kmpc_end_single,
585 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
586 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
587 // kmp_routine_entry_t *task_entry);
588 OMPRTL__kmpc_omp_task_alloc,
589 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
590 // new_task);
591 OMPRTL__kmpc_omp_task,
592 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
593 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
594 // kmp_int32 didit);
595 OMPRTL__kmpc_copyprivate,
596 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
597 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
598 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
599 OMPRTL__kmpc_reduce,
600 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
601 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
602 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
603 // *lck);
604 OMPRTL__kmpc_reduce_nowait,
605 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
606 // kmp_critical_name *lck);
607 OMPRTL__kmpc_end_reduce,
608 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
609 // kmp_critical_name *lck);
610 OMPRTL__kmpc_end_reduce_nowait,
611 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
612 // kmp_task_t * new_task);
613 OMPRTL__kmpc_omp_task_begin_if0,
614 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
615 // kmp_task_t * new_task);
616 OMPRTL__kmpc_omp_task_complete_if0,
617 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
618 OMPRTL__kmpc_ordered,
619 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
620 OMPRTL__kmpc_end_ordered,
621 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
622 // global_tid);
623 OMPRTL__kmpc_omp_taskwait,
624 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
625 OMPRTL__kmpc_taskgroup,
626 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
627 OMPRTL__kmpc_end_taskgroup,
628 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
629 // int proc_bind);
630 OMPRTL__kmpc_push_proc_bind,
631 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
632 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
633 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
634 OMPRTL__kmpc_omp_task_with_deps,
635 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
636 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
637 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
638 OMPRTL__kmpc_omp_wait_deps,
639 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
640 // global_tid, kmp_int32 cncl_kind);
641 OMPRTL__kmpc_cancellationpoint,
642 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
643 // kmp_int32 cncl_kind);
644 OMPRTL__kmpc_cancel,
645 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
646 // kmp_int32 num_teams, kmp_int32 thread_limit);
647 OMPRTL__kmpc_push_num_teams,
648 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
649 // microtask, ...);
650 OMPRTL__kmpc_fork_teams,
651 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
652 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
653 // sched, kmp_uint64 grainsize, void *task_dup);
654 OMPRTL__kmpc_taskloop,
655 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
656 // num_dims, struct kmp_dim *dims);
657 OMPRTL__kmpc_doacross_init,
658 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
659 OMPRTL__kmpc_doacross_fini,
660 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
661 // *vec);
662 OMPRTL__kmpc_doacross_post,
663 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
664 // *vec);
665 OMPRTL__kmpc_doacross_wait,
666 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
667 // *data);
668 OMPRTL__kmpc_task_reduction_init,
669 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
670 // *d);
671 OMPRTL__kmpc_task_reduction_get_th_data,
672
673 //
674 // Offloading related calls
675 //
676 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
677 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
678 // *arg_types);
679 OMPRTL__tgt_target,
680 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
681 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
682 // *arg_types);
683 OMPRTL__tgt_target_nowait,
684 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
685 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
686 // *arg_types, int32_t num_teams, int32_t thread_limit);
687 OMPRTL__tgt_target_teams,
688 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
689 // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
690 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
691 OMPRTL__tgt_target_teams_nowait,
692 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
693 OMPRTL__tgt_register_lib,
694 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
695 OMPRTL__tgt_unregister_lib,
696 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
697 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
698 OMPRTL__tgt_target_data_begin,
699 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
700 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
701 // *arg_types);
702 OMPRTL__tgt_target_data_begin_nowait,
703 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
704 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
705 OMPRTL__tgt_target_data_end,
706 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
707 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
708 // *arg_types);
709 OMPRTL__tgt_target_data_end_nowait,
710 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
711 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
712 OMPRTL__tgt_target_data_update,
713 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
714 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
715 // *arg_types);
716 OMPRTL__tgt_target_data_update_nowait,
717};
718
719/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
720/// region.
721class CleanupTy final : public EHScopeStack::Cleanup {
722 PrePostActionTy *Action;
723
724public:
725 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
726 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
727 if (!CGF.HaveInsertPoint())
728 return;
729 Action->Exit(CGF);
730 }
731};
732
733} // anonymous namespace
734
735void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
736 CodeGenFunction::RunCleanupsScope Scope(CGF);
737 if (PrePostAction) {
9
Taking false branch
738 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
739 Callback(CodeGen, CGF, *PrePostAction);
740 } else {
741 PrePostActionTy Action;
742 Callback(CodeGen, CGF, Action);
10
Calling 'RegionCodeGenTy::CallbackFn'
743 }
744}
745
746/// Check if the combiner is a call to UDR combiner and if it is so return the
747/// UDR decl used for reduction.
748static const OMPDeclareReductionDecl *
749getReductionInit(const Expr *ReductionOp) {
750 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
751 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
752 if (const auto *DRE =
753 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
754 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
755 return DRD;
756 return nullptr;
757}
758
759static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
760 const OMPDeclareReductionDecl *DRD,
761 const Expr *InitOp,
762 Address Private, Address Original,
763 QualType Ty) {
764 if (DRD->getInitializer()) {
765 std::pair<llvm::Function *, llvm::Function *> Reduction =
766 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
767 const auto *CE = cast<CallExpr>(InitOp);
768 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
769 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
770 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
771 const auto *LHSDRE =
772 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
773 const auto *RHSDRE =
774 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
775 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
776 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
777 [=]() { return Private; });
778 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
779 [=]() { return Original; });
780 (void)PrivateScope.Privatize();
781 RValue Func = RValue::get(Reduction.second);
782 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
783 CGF.EmitIgnoredExpr(InitOp);
784 } else {
785 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
786 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
787 auto *GV = new llvm::GlobalVariable(
788 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
789 llvm::GlobalValue::PrivateLinkage, Init, Name);
790 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
791 RValue InitRVal;
792 switch (CGF.getEvaluationKind(Ty)) {
793 case TEK_Scalar:
794 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
795 break;
796 case TEK_Complex:
797 InitRVal =
798 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
799 break;
800 case TEK_Aggregate:
801 InitRVal = RValue::getAggregate(LV.getAddress());
802 break;
803 }
804 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
805 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
806 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
807 /*IsInitializer=*/false);
808 }
809}
810
811/// Emit initialization of arrays of complex types.
812/// \param DestAddr Address of the array.
813/// \param Type Type of array.
814/// \param Init Initial expression of array.
815/// \param SrcAddr Address of the original array.
816static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
817 QualType Type, bool EmitDeclareReductionInit,
818 const Expr *Init,
819 const OMPDeclareReductionDecl *DRD,
820 Address SrcAddr = Address::invalid()) {
821 // Perform element-by-element initialization.
822 QualType ElementTy;
823
824 // Drill down to the base element type on both arrays.
825 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
826 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
827 DestAddr =
828 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
829 if (DRD)
830 SrcAddr =
831 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
832
833 llvm::Value *SrcBegin = nullptr;
834 if (DRD)
835 SrcBegin = SrcAddr.getPointer();
836 llvm::Value *DestBegin = DestAddr.getPointer();
837 // Cast from pointer to array type to pointer to single element.
838 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
839 // The basic structure here is a while-do loop.
840 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
841 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
842 llvm::Value *IsEmpty =
843 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
844 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
845
846 // Enter the loop body, making that address the current address.
847 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
848 CGF.EmitBlock(BodyBB);
849
850 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
851
852 llvm::PHINode *SrcElementPHI = nullptr;
853 Address SrcElementCurrent = Address::invalid();
854 if (DRD) {
855 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
856 "omp.arraycpy.srcElementPast");
857 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
858 SrcElementCurrent =
859 Address(SrcElementPHI,
860 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
861 }
862 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
863 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
864 DestElementPHI->addIncoming(DestBegin, EntryBB);
865 Address DestElementCurrent =
866 Address(DestElementPHI,
867 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
868
869 // Emit copy.
870 {
871 CodeGenFunction::RunCleanupsScope InitScope(CGF);
872 if (EmitDeclareReductionInit) {
873 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
874 SrcElementCurrent, ElementTy);
875 } else
876 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
877 /*IsInitializer=*/false);
878 }
879
880 if (DRD) {
881 // Shift the address forward by one element.
882 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
883 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
884 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
885 }
886
887 // Shift the address forward by one element.
888 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
889 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
890 // Check whether we've reached the end.
891 llvm::Value *Done =
892 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
893 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
894 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
895
896 // Done.
897 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
898}
899
900LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
901 return CGF.EmitOMPSharedLValue(E);
902}
903
904LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
905 const Expr *E) {
906 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
907 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
908 return LValue();
909}
910
911void ReductionCodeGen::emitAggregateInitialization(
912 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
913 const OMPDeclareReductionDecl *DRD) {
914 // Emit VarDecl with copy init for arrays.
915 // Get the address of the original variable captured in current
916 // captured region.
917 const auto *PrivateVD =
918 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
919 bool EmitDeclareReductionInit =
920 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
921 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
922 EmitDeclareReductionInit,
923 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
924 : PrivateVD->getInit(),
925 DRD, SharedLVal.getAddress());
926}
927
928ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
929 ArrayRef<const Expr *> Privates,
930 ArrayRef<const Expr *> ReductionOps) {
931 ClausesData.reserve(Shareds.size());
932 SharedAddresses.reserve(Shareds.size());
933 Sizes.reserve(Shareds.size());
934 BaseDecls.reserve(Shareds.size());
935 auto IPriv = Privates.begin();
936 auto IRed = ReductionOps.begin();
937 for (const Expr *Ref : Shareds) {
938 ClausesData.emplace_back(Ref, *IPriv, *IRed);
939 std::advance(IPriv, 1);
940 std::advance(IRed, 1);
941 }
942}
943
944void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
945 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 946, __PRETTY_FUNCTION__))
946 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 946, __PRETTY_FUNCTION__))
;
947 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
948 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
949 SharedAddresses.emplace_back(First, Second);
950}
951
952void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
953 const auto *PrivateVD =
954 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
955 QualType PrivateType = PrivateVD->getType();
956 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
957 if (!PrivateType->isVariablyModifiedType()) {
958 Sizes.emplace_back(
959 CGF.getTypeSize(
960 SharedAddresses[N].first.getType().getNonReferenceType()),
961 nullptr);
962 return;
963 }
964 llvm::Value *Size;
965 llvm::Value *SizeInChars;
966 auto *ElemType =
967 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
968 ->getElementType();
969 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
970 if (AsArraySection) {
971 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
972 SharedAddresses[N].first.getPointer());
973 Size = CGF.Builder.CreateNUWAdd(
974 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
975 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
976 } else {
977 SizeInChars = CGF.getTypeSize(
978 SharedAddresses[N].first.getType().getNonReferenceType());
979 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
980 }
981 Sizes.emplace_back(SizeInChars, Size);
982 CodeGenFunction::OpaqueValueMapping OpaqueMap(
983 CGF,
984 cast<OpaqueValueExpr>(
985 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
986 RValue::get(Size));
987 CGF.EmitVariablyModifiedType(PrivateType);
988}
989
990void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
991 llvm::Value *Size) {
992 const auto *PrivateVD =
993 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
994 QualType PrivateType = PrivateVD->getType();
995 if (!PrivateType->isVariablyModifiedType()) {
996 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 998, __PRETTY_FUNCTION__))
997 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 998, __PRETTY_FUNCTION__))
998 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 998, __PRETTY_FUNCTION__))
;
999 return;
1000 }
1001 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1002 CGF,
1003 cast<OpaqueValueExpr>(
1004 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1005 RValue::get(Size));
1006 CGF.EmitVariablyModifiedType(PrivateType);
1007}
1008
1009void ReductionCodeGen::emitInitialization(
1010 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1011 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1012 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1012, __PRETTY_FUNCTION__))
;
1013 const auto *PrivateVD =
1014 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1015 const OMPDeclareReductionDecl *DRD =
1016 getReductionInit(ClausesData[N].ReductionOp);
1017 QualType PrivateType = PrivateVD->getType();
1018 PrivateAddr = CGF.Builder.CreateElementBitCast(
1019 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1020 QualType SharedType = SharedAddresses[N].first.getType();
1021 SharedLVal = CGF.MakeAddrLValue(
1022 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1023 CGF.ConvertTypeForMem(SharedType)),
1024 SharedType, SharedAddresses[N].first.getBaseInfo(),
1025 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1026 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1027 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1028 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1029 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1030 PrivateAddr, SharedLVal.getAddress(),
1031 SharedLVal.getType());
1032 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1033 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1034 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1035 PrivateVD->getType().getQualifiers(),
1036 /*IsInitializer=*/false);
1037 }
1038}
1039
1040bool ReductionCodeGen::needCleanups(unsigned N) {
1041 const auto *PrivateVD =
1042 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1043 QualType PrivateType = PrivateVD->getType();
1044 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1045 return DTorKind != QualType::DK_none;
1046}
1047
1048void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1049 Address PrivateAddr) {
1050 const auto *PrivateVD =
1051 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1052 QualType PrivateType = PrivateVD->getType();
1053 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1054 if (needCleanups(N)) {
1055 PrivateAddr = CGF.Builder.CreateElementBitCast(
1056 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1057 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1058 }
1059}
1060
1061static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1062 LValue BaseLV) {
1063 BaseTy = BaseTy.getNonReferenceType();
1064 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1065 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1066 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1067 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1068 } else {
1069 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1070 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1071 }
1072 BaseTy = BaseTy->getPointeeType();
1073 }
1074 return CGF.MakeAddrLValue(
1075 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1076 CGF.ConvertTypeForMem(ElTy)),
1077 BaseLV.getType(), BaseLV.getBaseInfo(),
1078 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1079}
1080
1081static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1082 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1083 llvm::Value *Addr) {
1084 Address Tmp = Address::invalid();
1085 Address TopTmp = Address::invalid();
1086 Address MostTopTmp = Address::invalid();
1087 BaseTy = BaseTy.getNonReferenceType();
1088 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1089 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1090 Tmp = CGF.CreateMemTemp(BaseTy);
1091 if (TopTmp.isValid())
1092 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1093 else
1094 MostTopTmp = Tmp;
1095 TopTmp = Tmp;
1096 BaseTy = BaseTy->getPointeeType();
1097 }
1098 llvm::Type *Ty = BaseLVType;
1099 if (Tmp.isValid())
1100 Ty = Tmp.getElementType();
1101 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1102 if (Tmp.isValid()) {
1103 CGF.Builder.CreateStore(Addr, Tmp);
1104 return MostTopTmp;
1105 }
1106 return Address(Addr, BaseLVAlignment);
1107}
1108
1109static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1110 const VarDecl *OrigVD = nullptr;
1111 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1112 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1113 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1114 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1115 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1116 Base = TempASE->getBase()->IgnoreParenImpCasts();
1117 DE = cast<DeclRefExpr>(Base);
1118 OrigVD = cast<VarDecl>(DE->getDecl());
1119 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1120 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1121 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1122 Base = TempASE->getBase()->IgnoreParenImpCasts();
1123 DE = cast<DeclRefExpr>(Base);
1124 OrigVD = cast<VarDecl>(DE->getDecl());
1125 }
1126 return OrigVD;
1127}
1128
1129Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1130 Address PrivateAddr) {
1131 const DeclRefExpr *DE;
1132 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1133 BaseDecls.emplace_back(OrigVD);
1134 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1135 LValue BaseLValue =
1136 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1137 OriginalBaseLValue);
1138 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1139 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1140 llvm::Value *PrivatePointer =
1141 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1142 PrivateAddr.getPointer(),
1143 SharedAddresses[N].first.getAddress().getType());
1144 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1145 return castToBase(CGF, OrigVD->getType(),
1146 SharedAddresses[N].first.getType(),
1147 OriginalBaseLValue.getAddress().getType(),
1148 OriginalBaseLValue.getAlignment(), Ptr);
1149 }
1150 BaseDecls.emplace_back(
1151 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1152 return PrivateAddr;
1153}
1154
1155bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1156 const OMPDeclareReductionDecl *DRD =
1157 getReductionInit(ClausesData[N].ReductionOp);
1158 return DRD && DRD->getInitializer();
1159}
1160
1161LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1162 return CGF.EmitLoadOfPointerLValue(
1163 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1164 getThreadIDVariable()->getType()->castAs<PointerType>());
1165}
1166
1167void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1168 if (!CGF.HaveInsertPoint())
1169 return;
1170 // 1.2.2 OpenMP Language Terminology
1171 // Structured block - An executable statement with a single entry at the
1172 // top and a single exit at the bottom.
1173 // The point of exit cannot be a branch out of the structured block.
1174 // longjmp() and throw() must not violate the entry/exit criteria.
1175 CGF.EHStack.pushTerminate();
1176 CodeGen(CGF);
1177 CGF.EHStack.popTerminate();
1178}
1179
1180LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1181 CodeGenFunction &CGF) {
1182 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1183 getThreadIDVariable()->getType(),
1184 AlignmentSource::Decl);
1185}
1186
1187static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1188 QualType FieldTy) {
1189 auto *Field = FieldDecl::Create(
1190 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1191 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1192 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1193 Field->setAccess(AS_public);
1194 DC->addDecl(Field);
1195 return Field;
1196}
1197
1198CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1199 StringRef Separator)
1200 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1201 OffloadEntriesInfoManager(CGM) {
1202 ASTContext &C = CGM.getContext();
1203 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1204 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1205 RD->startDefinition();
1206 // reserved_1
1207 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1208 // flags
1209 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1210 // reserved_2
1211 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1212 // reserved_3
1213 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1214 // psource
1215 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1216 RD->completeDefinition();
1217 IdentQTy = C.getRecordType(RD);
1218 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1219 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1220
1221 loadOffloadInfoMetadata();
1222}
1223
1224void CGOpenMPRuntime::clear() {
1225 InternalVars.clear();
1226 // Clean non-target variable declarations possibly used only in debug info.
1227 for (const auto &Data : EmittedNonTargetVariables) {
1228 if (!Data.getValue().pointsToAliveValue())
1229 continue;
1230 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1231 if (!GV)
1232 continue;
1233 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1234 continue;
1235 GV->eraseFromParent();
1236 }
1237}
1238
1239std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1240 SmallString<128> Buffer;
1241 llvm::raw_svector_ostream OS(Buffer);
1242 StringRef Sep = FirstSeparator;
1243 for (StringRef Part : Parts) {
1244 OS << Sep << Part;
1245 Sep = Separator;
1246 }
1247 return OS.str();
1248}
1249
1250static llvm::Function *
1251emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1252 const Expr *CombinerInitializer, const VarDecl *In,
1253 const VarDecl *Out, bool IsCombiner) {
1254 // void .omp_combiner.(Ty *in, Ty *out);
1255 ASTContext &C = CGM.getContext();
1256 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1257 FunctionArgList Args;
1258 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1259 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1260 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1261 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1262 Args.push_back(&OmpOutParm);
1263 Args.push_back(&OmpInParm);
1264 const CGFunctionInfo &FnInfo =
1265 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1266 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1267 std::string Name = CGM.getOpenMPRuntime().getName(
1268 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1269 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1270 Name, &CGM.getModule());
1271 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1272 Fn->removeFnAttr(llvm::Attribute::NoInline);
1273 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1274 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1275 CodeGenFunction CGF(CGM);
1276 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1277 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1278 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1279 Out->getLocation());
1280 CodeGenFunction::OMPPrivateScope Scope(CGF);
1281 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1282 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1283 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1284 .getAddress();
1285 });
1286 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1287 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1288 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1289 .getAddress();
1290 });
1291 (void)Scope.Privatize();
1292 if (!IsCombiner && Out->hasInit() &&
1293 !CGF.isTrivialInitializer(Out->getInit())) {
1294 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1295 Out->getType().getQualifiers(),
1296 /*IsInitializer=*/true);
1297 }
1298 if (CombinerInitializer)
1299 CGF.EmitIgnoredExpr(CombinerInitializer);
1300 Scope.ForceCleanup();
1301 CGF.FinishFunction();
1302 return Fn;
1303}
1304
1305void CGOpenMPRuntime::emitUserDefinedReduction(
1306 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1307 if (UDRMap.count(D) > 0)
1308 return;
1309 llvm::Function *Combiner = emitCombinerOrInitializer(
1310 CGM, D->getType(), D->getCombiner(),
1311 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1312 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1313 /*IsCombiner=*/true);
1314 llvm::Function *Initializer = nullptr;
1315 if (const Expr *Init = D->getInitializer()) {
1316 Initializer = emitCombinerOrInitializer(
1317 CGM, D->getType(),
1318 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1319 : nullptr,
1320 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1321 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1322 /*IsCombiner=*/false);
1323 }
1324 UDRMap.try_emplace(D, Combiner, Initializer);
1325 if (CGF) {
1326 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1327 Decls.second.push_back(D);
1328 }
1329}
1330
1331std::pair<llvm::Function *, llvm::Function *>
1332CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1333 auto I = UDRMap.find(D);
1334 if (I != UDRMap.end())
1335 return I->second;
1336 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1337 return UDRMap.lookup(D);
1338}
1339
1340static llvm::Value *emitParallelOrTeamsOutlinedFunction(
1341 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1342 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1343 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1344 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1345, __PRETTY_FUNCTION__))
1345 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1345, __PRETTY_FUNCTION__))
;
1346 CodeGenFunction CGF(CGM, true);
1347 bool HasCancel = false;
1348 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1349 HasCancel = OPD->hasCancel();
1350 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1351 HasCancel = OPSD->hasCancel();
1352 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1353 HasCancel = OPFD->hasCancel();
1354 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1355 HasCancel = OPFD->hasCancel();
1356 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1357 HasCancel = OPFD->hasCancel();
1358 else if (const auto *OPFD =
1359 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1360 HasCancel = OPFD->hasCancel();
1361 else if (const auto *OPFD =
1362 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1363 HasCancel = OPFD->hasCancel();
1364 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1365 HasCancel, OutlinedHelperName);
1366 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1367 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1368}
1369
1370llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction(
1371 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1372 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1373 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1374 return emitParallelOrTeamsOutlinedFunction(
1375 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1376}
1377
1378llvm::Value *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1379 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1380 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1381 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1382 return emitParallelOrTeamsOutlinedFunction(
1383 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1384}
1385
1386llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction(
1387 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1388 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1389 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1390 bool Tied, unsigned &NumberOfParts) {
1391 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1392 PrePostActionTy &) {
1393 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1394 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1395 llvm::Value *TaskArgs[] = {
1396 UpLoc, ThreadID,
1397 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1398 TaskTVar->getType()->castAs<PointerType>())
1399 .getPointer()};
1400 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1401 };
1402 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1403 UntiedCodeGen);
1404 CodeGen.setAction(Action);
1405 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1406, __PRETTY_FUNCTION__))
1406 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1406, __PRETTY_FUNCTION__))
;
1407 const OpenMPDirectiveKind Region =
1408 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1409 : OMPD_task;
1410 const CapturedStmt *CS = D.getCapturedStmt(Region);
1411 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1412 CodeGenFunction CGF(CGM, true);
1413 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1414 InnermostKind,
1415 TD ? TD->hasCancel() : false, Action);
1416 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1417 llvm::Value *Res = CGF.GenerateCapturedStmtFunction(*CS);
1418 if (!Tied)
1419 NumberOfParts = Action.getNumberOfParts();
1420 return Res;
1421}
1422
1423static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1424 const RecordDecl *RD, const CGRecordLayout &RL,
1425 ArrayRef<llvm::Constant *> Data) {
1426 llvm::StructType *StructTy = RL.getLLVMType();
1427 unsigned PrevIdx = 0;
1428 ConstantInitBuilder CIBuilder(CGM);
1429 auto DI = Data.begin();
1430 for (const FieldDecl *FD : RD->fields()) {
1431 unsigned Idx = RL.getLLVMFieldNo(FD);
1432 // Fill the alignment.
1433 for (unsigned I = PrevIdx; I < Idx; ++I)
1434 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1435 PrevIdx = Idx + 1;
1436 Fields.add(*DI);
1437 ++DI;
1438 }
1439}
1440
1441template <class... As>
1442static llvm::GlobalVariable *
1443createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1444 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1445 As &&... Args) {
1446 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1447 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1448 ConstantInitBuilder CIBuilder(CGM);
1449 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1450 buildStructValue(Fields, CGM, RD, RL, Data);
1451 return Fields.finishAndCreateGlobal(
1452 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1453 std::forward<As>(Args)...);
1454}
1455
1456template <typename T>
1457static void
1458createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1459 ArrayRef<llvm::Constant *> Data,
1460 T &Parent) {
1461 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1462 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1463 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1464 buildStructValue(Fields, CGM, RD, RL, Data);
1465 Fields.finishAndAddTo(Parent);
1466}
1467
1468Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1469 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1470 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1471 FlagsTy FlagsKey(Flags, Reserved2Flags);
1472 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1473 if (!Entry) {
1474 if (!DefaultOpenMPPSource) {
1475 // Initialize default location for psource field of ident_t structure of
1476 // all ident_t objects. Format is ";file;function;line;column;;".
1477 // Taken from
1478 // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
1479 DefaultOpenMPPSource =
1480 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1481 DefaultOpenMPPSource =
1482 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1483 }
1484
1485 llvm::Constant *Data[] = {
1486 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1487 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1488 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1489 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1490 llvm::GlobalValue *DefaultOpenMPLocation =
1491 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1492 llvm::GlobalValue::PrivateLinkage);
1493 DefaultOpenMPLocation->setUnnamedAddr(
1494 llvm::GlobalValue::UnnamedAddr::Global);
1495
1496 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1497 }
1498 return Address(Entry, Align);
1499}
1500
1501void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1502 bool AtCurrentPoint) {
1503 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1504 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1504, __PRETTY_FUNCTION__))
;
1505
1506 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1507 if (AtCurrentPoint) {
1508 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1509 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1510 } else {
1511 Elem.second.ServiceInsertPt =
1512 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1513 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1514 }
1515}
1516
1517void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1518 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1519 if (Elem.second.ServiceInsertPt) {
1520 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1521 Elem.second.ServiceInsertPt = nullptr;
1522 Ptr->eraseFromParent();
1523 }
1524}
1525
1526llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1527 SourceLocation Loc,
1528 unsigned Flags) {
1529 Flags |= OMP_IDENT_KMPC;
1530 // If no debug info is generated - return global default location.
1531 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1532 Loc.isInvalid())
1533 return getOrCreateDefaultLocation(Flags).getPointer();
1534
1535 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1535, __PRETTY_FUNCTION__))
;
1536
1537 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1538 Address LocValue = Address::invalid();
1539 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1540 if (I != OpenMPLocThreadIDMap.end())
1541 LocValue = Address(I->second.DebugLoc, Align);
1542
1543 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1544 // GetOpenMPThreadID was called before this routine.
1545 if (!LocValue.isValid()) {
1546 // Generate "ident_t .kmpc_loc.addr;"
1547 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1548 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1549 Elem.second.DebugLoc = AI.getPointer();
1550 LocValue = AI;
1551
1552 if (!Elem.second.ServiceInsertPt)
1553 setLocThreadIdInsertPt(CGF);
1554 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1555 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1556 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1557 CGF.getTypeSize(IdentQTy));
1558 }
1559
1560 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1561 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1562 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1563 LValue PSource =
1564 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1565
1566 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1567 if (OMPDebugLoc == nullptr) {
1568 SmallString<128> Buffer2;
1569 llvm::raw_svector_ostream OS2(Buffer2);
1570 // Build debug location
1571 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1572 OS2 << ";" << PLoc.getFilename() << ";";
1573 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1574 OS2 << FD->getQualifiedNameAsString();
1575 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1576 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1577 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1578 }
1579 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1580 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1581
1582 // Our callers always pass this to a runtime function, so for
1583 // convenience, go ahead and return a naked pointer.
1584 return LocValue.getPointer();
1585}
1586
1587llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1588 SourceLocation Loc) {
1589 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1589, __PRETTY_FUNCTION__))
;
1590
1591 llvm::Value *ThreadID = nullptr;
1592 // Check whether we've already cached a load of the thread id in this
1593 // function.
1594 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1595 if (I != OpenMPLocThreadIDMap.end()) {
1596 ThreadID = I->second.ThreadID;
1597 if (ThreadID != nullptr)
1598 return ThreadID;
1599 }
1600 // If exceptions are enabled, do not use parameter to avoid possible crash.
1601 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1602 !CGF.getLangOpts().CXXExceptions ||
1603 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1604 if (auto *OMPRegionInfo =
1605 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1606 if (OMPRegionInfo->getThreadIDVariable()) {
1607 // Check if this an outlined function with thread id passed as argument.
1608 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1609 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1610 // If value loaded in entry block, cache it and use it everywhere in
1611 // function.
1612 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1613 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1614 Elem.second.ThreadID = ThreadID;
1615 }
1616 return ThreadID;
1617 }
1618 }
1619 }
1620
1621 // This is not an outlined function region - need to call __kmpc_int32
1622 // kmpc_global_thread_num(ident_t *loc).
1623 // Generate thread id value and cache this value for use across the
1624 // function.
1625 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1626 if (!Elem.second.ServiceInsertPt)
1627 setLocThreadIdInsertPt(CGF);
1628 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1629 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1630 llvm::CallInst *Call = CGF.Builder.CreateCall(
1631 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1632 emitUpdateLocation(CGF, Loc));
1633 Call->setCallingConv(CGF.getRuntimeCC());
1634 Elem.second.ThreadID = Call;
1635 return Call;
1636}
1637
1638void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1639 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1639, __PRETTY_FUNCTION__))
;
1640 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1641 clearLocThreadIdInsertPt(CGF);
1642 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1643 }
1644 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1645 for(auto *D : FunctionUDRMap[CGF.CurFn])
1646 UDRMap.erase(D);
1647 FunctionUDRMap.erase(CGF.CurFn);
1648 }
1649}
1650
1651llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1652 return IdentTy->getPointerTo();
1653}
1654
1655llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1656 if (!Kmpc_MicroTy) {
1657 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1658 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1659 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1660 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1661 }
1662 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1663}
1664
1665llvm::Constant *
1666CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1667 llvm::Constant *RTLFn = nullptr;
1668 switch (static_cast<OpenMPRTLFunction>(Function)) {
1669 case OMPRTL__kmpc_fork_call: {
1670 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1671 // microtask, ...);
1672 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1673 getKmpc_MicroPointerTy()};
1674 auto *FnTy =
1675 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1676 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1677 break;
1678 }
1679 case OMPRTL__kmpc_global_thread_num: {
1680 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1681 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1682 auto *FnTy =
1683 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1684 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1685 break;
1686 }
1687 case OMPRTL__kmpc_threadprivate_cached: {
1688 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1689 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1690 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1691 CGM.VoidPtrTy, CGM.SizeTy,
1692 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1693 auto *FnTy =
1694 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1695 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1696 break;
1697 }
1698 case OMPRTL__kmpc_critical: {
1699 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1700 // kmp_critical_name *crit);
1701 llvm::Type *TypeParams[] = {
1702 getIdentTyPointerTy(), CGM.Int32Ty,
1703 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1704 auto *FnTy =
1705 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1706 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1707 break;
1708 }
1709 case OMPRTL__kmpc_critical_with_hint: {
1710 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1711 // kmp_critical_name *crit, uintptr_t hint);
1712 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1713 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1714 CGM.IntPtrTy};
1715 auto *FnTy =
1716 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1717 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1718 break;
1719 }
1720 case OMPRTL__kmpc_threadprivate_register: {
1721 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1722 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1723 // typedef void *(*kmpc_ctor)(void *);
1724 auto *KmpcCtorTy =
1725 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1726 /*isVarArg*/ false)->getPointerTo();
1727 // typedef void *(*kmpc_cctor)(void *, void *);
1728 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1729 auto *KmpcCopyCtorTy =
1730 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1731 /*isVarArg*/ false)
1732 ->getPointerTo();
1733 // typedef void (*kmpc_dtor)(void *);
1734 auto *KmpcDtorTy =
1735 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1736 ->getPointerTo();
1737 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1738 KmpcCopyCtorTy, KmpcDtorTy};
1739 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1740 /*isVarArg*/ false);
1741 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1742 break;
1743 }
1744 case OMPRTL__kmpc_end_critical: {
1745 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1746 // kmp_critical_name *crit);
1747 llvm::Type *TypeParams[] = {
1748 getIdentTyPointerTy(), CGM.Int32Ty,
1749 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1750 auto *FnTy =
1751 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1752 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1753 break;
1754 }
1755 case OMPRTL__kmpc_cancel_barrier: {
1756 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1757 // global_tid);
1758 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1759 auto *FnTy =
1760 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1761 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1762 break;
1763 }
1764 case OMPRTL__kmpc_barrier: {
1765 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1766 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1767 auto *FnTy =
1768 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1769 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1770 break;
1771 }
1772 case OMPRTL__kmpc_for_static_fini: {
1773 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1774 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1775 auto *FnTy =
1776 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1777 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1778 break;
1779 }
1780 case OMPRTL__kmpc_push_num_threads: {
1781 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1782 // kmp_int32 num_threads)
1783 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1784 CGM.Int32Ty};
1785 auto *FnTy =
1786 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1787 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1788 break;
1789 }
1790 case OMPRTL__kmpc_serialized_parallel: {
1791 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1792 // global_tid);
1793 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1794 auto *FnTy =
1795 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1796 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1797 break;
1798 }
1799 case OMPRTL__kmpc_end_serialized_parallel: {
1800 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1801 // global_tid);
1802 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1803 auto *FnTy =
1804 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1805 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1806 break;
1807 }
1808 case OMPRTL__kmpc_flush: {
1809 // Build void __kmpc_flush(ident_t *loc);
1810 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1811 auto *FnTy =
1812 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1813 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1814 break;
1815 }
1816 case OMPRTL__kmpc_master: {
1817 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1818 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1819 auto *FnTy =
1820 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1821 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1822 break;
1823 }
1824 case OMPRTL__kmpc_end_master: {
1825 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1826 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1827 auto *FnTy =
1828 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1829 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1830 break;
1831 }
1832 case OMPRTL__kmpc_omp_taskyield: {
1833 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1834 // int end_part);
1835 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1836 auto *FnTy =
1837 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1838 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1839 break;
1840 }
1841 case OMPRTL__kmpc_single: {
1842 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1843 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1844 auto *FnTy =
1845 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1846 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1847 break;
1848 }
1849 case OMPRTL__kmpc_end_single: {
1850 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1851 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1852 auto *FnTy =
1853 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1854 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1855 break;
1856 }
1857 case OMPRTL__kmpc_omp_task_alloc: {
1858 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1859 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1860 // kmp_routine_entry_t *task_entry);
1861 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1862, __PRETTY_FUNCTION__))
1862 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1862, __PRETTY_FUNCTION__))
;
1863 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1864 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1865 // Return void * and then cast to particular kmp_task_t type.
1866 auto *FnTy =
1867 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1868 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1869 break;
1870 }
1871 case OMPRTL__kmpc_omp_task: {
1872 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1873 // *new_task);
1874 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1875 CGM.VoidPtrTy};
1876 auto *FnTy =
1877 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1878 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
1879 break;
1880 }
1881 case OMPRTL__kmpc_copyprivate: {
1882 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
1883 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
1884 // kmp_int32 didit);
1885 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1886 auto *CpyFnTy =
1887 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
1888 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
1889 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1890 CGM.Int32Ty};
1891 auto *FnTy =
1892 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1893 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
1894 break;
1895 }
1896 case OMPRTL__kmpc_reduce: {
1897 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
1898 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
1899 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
1900 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1901 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1902 /*isVarArg=*/false);
1903 llvm::Type *TypeParams[] = {
1904 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1905 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1906 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1907 auto *FnTy =
1908 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1909 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
1910 break;
1911 }
1912 case OMPRTL__kmpc_reduce_nowait: {
1913 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
1914 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
1915 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
1916 // *lck);
1917 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1918 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1919 /*isVarArg=*/false);
1920 llvm::Type *TypeParams[] = {
1921 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1922 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1923 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1924 auto *FnTy =
1925 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1926 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
1927 break;
1928 }
1929 case OMPRTL__kmpc_end_reduce: {
1930 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
1931 // kmp_critical_name *lck);
1932 llvm::Type *TypeParams[] = {
1933 getIdentTyPointerTy(), CGM.Int32Ty,
1934 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1935 auto *FnTy =
1936 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1937 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
1938 break;
1939 }
1940 case OMPRTL__kmpc_end_reduce_nowait: {
1941 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
1942 // kmp_critical_name *lck);
1943 llvm::Type *TypeParams[] = {
1944 getIdentTyPointerTy(), CGM.Int32Ty,
1945 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1946 auto *FnTy =
1947 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1948 RTLFn =
1949 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
1950 break;
1951 }
1952 case OMPRTL__kmpc_omp_task_begin_if0: {
1953 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1954 // *new_task);
1955 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1956 CGM.VoidPtrTy};
1957 auto *FnTy =
1958 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1959 RTLFn =
1960 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
1961 break;
1962 }
1963 case OMPRTL__kmpc_omp_task_complete_if0: {
1964 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1965 // *new_task);
1966 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1967 CGM.VoidPtrTy};
1968 auto *FnTy =
1969 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1970 RTLFn = CGM.CreateRuntimeFunction(FnTy,
1971 /*Name=*/"__kmpc_omp_task_complete_if0");
1972 break;
1973 }
1974 case OMPRTL__kmpc_ordered: {
1975 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
1976 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1977 auto *FnTy =
1978 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1979 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
1980 break;
1981 }
1982 case OMPRTL__kmpc_end_ordered: {
1983 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
1984 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1985 auto *FnTy =
1986 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1987 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
1988 break;
1989 }
1990 case OMPRTL__kmpc_omp_taskwait: {
1991 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
1992 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1993 auto *FnTy =
1994 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1995 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
1996 break;
1997 }
1998 case OMPRTL__kmpc_taskgroup: {
1999 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2000 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2001 auto *FnTy =
2002 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2003 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2004 break;
2005 }
2006 case OMPRTL__kmpc_end_taskgroup: {
2007 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2008 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2009 auto *FnTy =
2010 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2011 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2012 break;
2013 }
2014 case OMPRTL__kmpc_push_proc_bind: {
2015 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2016 // int proc_bind)
2017 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2018 auto *FnTy =
2019 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2020 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2021 break;
2022 }
2023 case OMPRTL__kmpc_omp_task_with_deps: {
2024 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2025 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2026 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2027 llvm::Type *TypeParams[] = {
2028 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2029 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2030 auto *FnTy =
2031 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2032 RTLFn =
2033 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2034 break;
2035 }
2036 case OMPRTL__kmpc_omp_wait_deps: {
2037 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2038 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2039 // kmp_depend_info_t *noalias_dep_list);
2040 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2041 CGM.Int32Ty, CGM.VoidPtrTy,
2042 CGM.Int32Ty, CGM.VoidPtrTy};
2043 auto *FnTy =
2044 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2045 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2046 break;
2047 }
2048 case OMPRTL__kmpc_cancellationpoint: {
2049 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2050 // global_tid, kmp_int32 cncl_kind)
2051 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2052 auto *FnTy =
2053 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2054 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2055 break;
2056 }
2057 case OMPRTL__kmpc_cancel: {
2058 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2059 // kmp_int32 cncl_kind)
2060 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2061 auto *FnTy =
2062 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2063 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2064 break;
2065 }
2066 case OMPRTL__kmpc_push_num_teams: {
2067 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2068 // kmp_int32 num_teams, kmp_int32 num_threads)
2069 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2070 CGM.Int32Ty};
2071 auto *FnTy =
2072 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2073 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2074 break;
2075 }
2076 case OMPRTL__kmpc_fork_teams: {
2077 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2078 // microtask, ...);
2079 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2080 getKmpc_MicroPointerTy()};
2081 auto *FnTy =
2082 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2083 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2084 break;
2085 }
2086 case OMPRTL__kmpc_taskloop: {
2087 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2088 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2089 // sched, kmp_uint64 grainsize, void *task_dup);
2090 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2091 CGM.IntTy,
2092 CGM.VoidPtrTy,
2093 CGM.IntTy,
2094 CGM.Int64Ty->getPointerTo(),
2095 CGM.Int64Ty->getPointerTo(),
2096 CGM.Int64Ty,
2097 CGM.IntTy,
2098 CGM.IntTy,
2099 CGM.Int64Ty,
2100 CGM.VoidPtrTy};
2101 auto *FnTy =
2102 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2103 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2104 break;
2105 }
2106 case OMPRTL__kmpc_doacross_init: {
2107 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2108 // num_dims, struct kmp_dim *dims);
2109 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2110 CGM.Int32Ty,
2111 CGM.Int32Ty,
2112 CGM.VoidPtrTy};
2113 auto *FnTy =
2114 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2115 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2116 break;
2117 }
2118 case OMPRTL__kmpc_doacross_fini: {
2119 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2120 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2121 auto *FnTy =
2122 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2123 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2124 break;
2125 }
2126 case OMPRTL__kmpc_doacross_post: {
2127 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2128 // *vec);
2129 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2130 CGM.Int64Ty->getPointerTo()};
2131 auto *FnTy =
2132 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2133 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2134 break;
2135 }
2136 case OMPRTL__kmpc_doacross_wait: {
2137 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2138 // *vec);
2139 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2140 CGM.Int64Ty->getPointerTo()};
2141 auto *FnTy =
2142 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2143 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2144 break;
2145 }
2146 case OMPRTL__kmpc_task_reduction_init: {
2147 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2148 // *data);
2149 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2150 auto *FnTy =
2151 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2152 RTLFn =
2153 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2154 break;
2155 }
2156 case OMPRTL__kmpc_task_reduction_get_th_data: {
2157 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2158 // *d);
2159 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2160 auto *FnTy =
2161 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2162 RTLFn = CGM.CreateRuntimeFunction(
2163 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2164 break;
2165 }
2166 case OMPRTL__tgt_target: {
2167 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2168 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2169 // *arg_types);
2170 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2171 CGM.VoidPtrTy,
2172 CGM.Int32Ty,
2173 CGM.VoidPtrPtrTy,
2174 CGM.VoidPtrPtrTy,
2175 CGM.SizeTy->getPointerTo(),
2176 CGM.Int64Ty->getPointerTo()};
2177 auto *FnTy =
2178 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2179 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2180 break;
2181 }
2182 case OMPRTL__tgt_target_nowait: {
2183 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2184 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2185 // int64_t *arg_types);
2186 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2187 CGM.VoidPtrTy,
2188 CGM.Int32Ty,
2189 CGM.VoidPtrPtrTy,
2190 CGM.VoidPtrPtrTy,
2191 CGM.SizeTy->getPointerTo(),
2192 CGM.Int64Ty->getPointerTo()};
2193 auto *FnTy =
2194 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2195 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2196 break;
2197 }
2198 case OMPRTL__tgt_target_teams: {
2199 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2200 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2201 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2202 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2203 CGM.VoidPtrTy,
2204 CGM.Int32Ty,
2205 CGM.VoidPtrPtrTy,
2206 CGM.VoidPtrPtrTy,
2207 CGM.SizeTy->getPointerTo(),
2208 CGM.Int64Ty->getPointerTo(),
2209 CGM.Int32Ty,
2210 CGM.Int32Ty};
2211 auto *FnTy =
2212 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2213 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2214 break;
2215 }
2216 case OMPRTL__tgt_target_teams_nowait: {
2217 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2218 // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
2219 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2220 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2221 CGM.VoidPtrTy,
2222 CGM.Int32Ty,
2223 CGM.VoidPtrPtrTy,
2224 CGM.VoidPtrPtrTy,
2225 CGM.SizeTy->getPointerTo(),
2226 CGM.Int64Ty->getPointerTo(),
2227 CGM.Int32Ty,
2228 CGM.Int32Ty};
2229 auto *FnTy =
2230 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2231 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2232 break;
2233 }
2234 case OMPRTL__tgt_register_lib: {
2235 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2236 QualType ParamTy =
2237 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2238 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2239 auto *FnTy =
2240 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2241 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2242 break;
2243 }
2244 case OMPRTL__tgt_unregister_lib: {
2245 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2246 QualType ParamTy =
2247 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2248 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2249 auto *FnTy =
2250 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2251 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2252 break;
2253 }
2254 case OMPRTL__tgt_target_data_begin: {
2255 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2256 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2257 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2258 CGM.Int32Ty,
2259 CGM.VoidPtrPtrTy,
2260 CGM.VoidPtrPtrTy,
2261 CGM.SizeTy->getPointerTo(),
2262 CGM.Int64Ty->getPointerTo()};
2263 auto *FnTy =
2264 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2265 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2266 break;
2267 }
2268 case OMPRTL__tgt_target_data_begin_nowait: {
2269 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2270 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2271 // *arg_types);
2272 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2273 CGM.Int32Ty,
2274 CGM.VoidPtrPtrTy,
2275 CGM.VoidPtrPtrTy,
2276 CGM.SizeTy->getPointerTo(),
2277 CGM.Int64Ty->getPointerTo()};
2278 auto *FnTy =
2279 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2280 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2281 break;
2282 }
2283 case OMPRTL__tgt_target_data_end: {
2284 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2285 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2286 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2287 CGM.Int32Ty,
2288 CGM.VoidPtrPtrTy,
2289 CGM.VoidPtrPtrTy,
2290 CGM.SizeTy->getPointerTo(),
2291 CGM.Int64Ty->getPointerTo()};
2292 auto *FnTy =
2293 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2294 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2295 break;
2296 }
2297 case OMPRTL__tgt_target_data_end_nowait: {
2298 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2299 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2300 // *arg_types);
2301 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2302 CGM.Int32Ty,
2303 CGM.VoidPtrPtrTy,
2304 CGM.VoidPtrPtrTy,
2305 CGM.SizeTy->getPointerTo(),
2306 CGM.Int64Ty->getPointerTo()};
2307 auto *FnTy =
2308 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2309 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2310 break;
2311 }
2312 case OMPRTL__tgt_target_data_update: {
2313 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2314 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2315 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2316 CGM.Int32Ty,
2317 CGM.VoidPtrPtrTy,
2318 CGM.VoidPtrPtrTy,
2319 CGM.SizeTy->getPointerTo(),
2320 CGM.Int64Ty->getPointerTo()};
2321 auto *FnTy =
2322 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2323 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2324 break;
2325 }
2326 case OMPRTL__tgt_target_data_update_nowait: {
2327 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2328 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2329 // *arg_types);
2330 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2331 CGM.Int32Ty,
2332 CGM.VoidPtrPtrTy,
2333 CGM.VoidPtrPtrTy,
2334 CGM.SizeTy->getPointerTo(),
2335 CGM.Int64Ty->getPointerTo()};
2336 auto *FnTy =
2337 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2338 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2339 break;
2340 }
2341 }
2342 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2342, __PRETTY_FUNCTION__))
;
2343 return RTLFn;
2344}
2345
2346llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
2347 bool IVSigned) {
2348 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2349, __PRETTY_FUNCTION__))
2349 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2349, __PRETTY_FUNCTION__))
;
2350 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2351 : "__kmpc_for_static_init_4u")
2352 : (IVSigned ? "__kmpc_for_static_init_8"
2353 : "__kmpc_for_static_init_8u");
2354 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2355 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2356 llvm::Type *TypeParams[] = {
2357 getIdentTyPointerTy(), // loc
2358 CGM.Int32Ty, // tid
2359 CGM.Int32Ty, // schedtype
2360 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2361 PtrTy, // p_lower
2362 PtrTy, // p_upper
2363 PtrTy, // p_stride
2364 ITy, // incr
2365 ITy // chunk
2366 };
2367 auto *FnTy =
2368 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2369 return CGM.CreateRuntimeFunction(FnTy, Name);
2370}
2371
2372llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize,
2373 bool IVSigned) {
2374 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2375, __PRETTY_FUNCTION__))
2375 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2375, __PRETTY_FUNCTION__))
;
2376 StringRef Name =
2377 IVSize == 32
2378 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2379 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2380 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2381 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2382 CGM.Int32Ty, // tid
2383 CGM.Int32Ty, // schedtype
2384 ITy, // lower
2385 ITy, // upper
2386 ITy, // stride
2387 ITy // chunk
2388 };
2389 auto *FnTy =
2390 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2391 return CGM.CreateRuntimeFunction(FnTy, Name);
2392}
2393
2394llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
2395 bool IVSigned) {
2396 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2397, __PRETTY_FUNCTION__))
2397 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2397, __PRETTY_FUNCTION__))
;
2398 StringRef Name =
2399 IVSize == 32
2400 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2401 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2402 llvm::Type *TypeParams[] = {
2403 getIdentTyPointerTy(), // loc
2404 CGM.Int32Ty, // tid
2405 };
2406 auto *FnTy =
2407 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2408 return CGM.CreateRuntimeFunction(FnTy, Name);
2409}
2410
2411llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
2412 bool IVSigned) {
2413 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2414, __PRETTY_FUNCTION__))
2414 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2414, __PRETTY_FUNCTION__))
;
2415 StringRef Name =
2416 IVSize == 32
2417 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2418 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2419 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2420 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2421 llvm::Type *TypeParams[] = {
2422 getIdentTyPointerTy(), // loc
2423 CGM.Int32Ty, // tid
2424 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2425 PtrTy, // p_lower
2426 PtrTy, // p_upper
2427 PtrTy // p_stride
2428 };
2429 auto *FnTy =
2430 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2431 return CGM.CreateRuntimeFunction(FnTy, Name);
2432}
2433
2434Address CGOpenMPRuntime::getAddrOfDeclareTargetLink(const VarDecl *VD) {
2435 if (CGM.getLangOpts().OpenMPSimd)
2436 return Address::invalid();
2437 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2438 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2439 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Link) {
2440 SmallString<64> PtrName;
2441 {
2442 llvm::raw_svector_ostream OS(PtrName);
2443 OS << CGM.getMangledName(GlobalDecl(VD)) << "_decl_tgt_link_ptr";
2444 }
2445 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2446 if (!Ptr) {
2447 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2448 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2449 PtrName);
2450 if (!CGM.getLangOpts().OpenMPIsDevice) {
2451 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2452 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2453 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2454 }
2455 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ptr));
2456 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2457 }
2458 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2459 }
2460 return Address::invalid();
2461}
2462
2463llvm::Constant *
2464CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2465 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2466, __PRETTY_FUNCTION__))
2466 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2466, __PRETTY_FUNCTION__))
;
2467 // Lookup the entry, lazily creating it if necessary.
2468 std::string Suffix = getName({"cache", ""});
2469 return getOrCreateInternalVariable(
2470 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2471}
2472
2473Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2474 const VarDecl *VD,
2475 Address VDAddr,
2476 SourceLocation Loc) {
2477 if (CGM.getLangOpts().OpenMPUseTLS &&
2478 CGM.getContext().getTargetInfo().isTLSSupported())
2479 return VDAddr;
2480
2481 llvm::Type *VarTy = VDAddr.getElementType();
2482 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2483 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2484 CGM.Int8PtrTy),
2485 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2486 getOrCreateThreadPrivateCache(VD)};
2487 return Address(CGF.EmitRuntimeCall(
2488 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2489 VDAddr.getAlignment());
2490}
2491
2492void CGOpenMPRuntime::emitThreadPrivateVarInit(
2493 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2494 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2495 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2496 // library.
2497 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2498 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2499 OMPLoc);
2500 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2501 // to register constructor/destructor for variable.
2502 llvm::Value *Args[] = {
2503 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2504 Ctor, CopyCtor, Dtor};
2505 CGF.EmitRuntimeCall(
2506 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2507}
2508
2509llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2510 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2511 bool PerformInit, CodeGenFunction *CGF) {
2512 if (CGM.getLangOpts().OpenMPUseTLS &&
2513 CGM.getContext().getTargetInfo().isTLSSupported())
2514 return nullptr;
2515
2516 VD = VD->getDefinition(CGM.getContext());
2517 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2518 QualType ASTTy = VD->getType();
2519
2520 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2521 const Expr *Init = VD->getAnyInitializer();
2522 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2523 // Generate function that re-emits the declaration's initializer into the
2524 // threadprivate copy of the variable VD
2525 CodeGenFunction CtorCGF(CGM);
2526 FunctionArgList Args;
2527 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2528 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2529 ImplicitParamDecl::Other);
2530 Args.push_back(&Dst);
2531
2532 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2533 CGM.getContext().VoidPtrTy, Args);
2534 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2535 std::string Name = getName({"__kmpc_global_ctor_", ""});
2536 llvm::Function *Fn =
2537 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2538 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2539 Args, Loc, Loc);
2540 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2541 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2542 CGM.getContext().VoidPtrTy, Dst.getLocation());
2543 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2544 Arg = CtorCGF.Builder.CreateElementBitCast(
2545 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2546 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2547 /*IsInitializer=*/true);
2548 ArgVal = CtorCGF.EmitLoadOfScalar(
2549 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2550 CGM.getContext().VoidPtrTy, Dst.getLocation());
2551 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2552 CtorCGF.FinishFunction();
2553 Ctor = Fn;
2554 }
2555 if (VD->getType().isDestructedType() != QualType::DK_none) {
2556 // Generate function that emits destructor call for the threadprivate copy
2557 // of the variable VD
2558 CodeGenFunction DtorCGF(CGM);
2559 FunctionArgList Args;
2560 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2561 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2562 ImplicitParamDecl::Other);
2563 Args.push_back(&Dst);
2564
2565 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2566 CGM.getContext().VoidTy, Args);
2567 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2568 std::string Name = getName({"__kmpc_global_dtor_", ""});
2569 llvm::Function *Fn =
2570 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2571 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2572 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2573 Loc, Loc);
2574 // Create a scope with an artificial location for the body of this function.
2575 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2576 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2577 DtorCGF.GetAddrOfLocalVar(&Dst),
2578 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2579 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2580 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2581 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2582 DtorCGF.FinishFunction();
2583 Dtor = Fn;
2584 }
2585 // Do not emit init function if it is not required.
2586 if (!Ctor && !Dtor)
2587 return nullptr;
2588
2589 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2590 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2591 /*isVarArg=*/false)
2592 ->getPointerTo();
2593 // Copying constructor for the threadprivate variable.
2594 // Must be NULL - reserved by runtime, but currently it requires that this
2595 // parameter is always NULL. Otherwise it fires assertion.
2596 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2597 if (Ctor == nullptr) {
2598 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2599 /*isVarArg=*/false)
2600 ->getPointerTo();
2601 Ctor = llvm::Constant::getNullValue(CtorTy);
2602 }
2603 if (Dtor == nullptr) {
2604 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2605 /*isVarArg=*/false)
2606 ->getPointerTo();
2607 Dtor = llvm::Constant::getNullValue(DtorTy);
2608 }
2609 if (!CGF) {
2610 auto *InitFunctionTy =
2611 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2612 std::string Name = getName({"__omp_threadprivate_init_", ""});
2613 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2614 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2615 CodeGenFunction InitCGF(CGM);
2616 FunctionArgList ArgList;
2617 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2618 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2619 Loc, Loc);
2620 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2621 InitCGF.FinishFunction();
2622 return InitFunction;
2623 }
2624 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2625 }
2626 return nullptr;
2627}
2628
2629/// Obtain information that uniquely identifies a target entry. This
2630/// consists of the file and device IDs as well as line number associated with
2631/// the relevant entry source location.
2632static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2633 unsigned &DeviceID, unsigned &FileID,
2634 unsigned &LineNum) {
2635 SourceManager &SM = C.getSourceManager();
2636
2637 // The loc should be always valid and have a file ID (the user cannot use
2638 // #pragma directives in macros)
2639
2640 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2640, __PRETTY_FUNCTION__))
;
2641
2642 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2643 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2643, __PRETTY_FUNCTION__))
;
2644
2645 llvm::sys::fs::UniqueID ID;
2646 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2647 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2648 << PLoc.getFilename() << EC.message();
2649
2650 DeviceID = ID.getDevice();
2651 FileID = ID.getFile();
2652 LineNum = PLoc.getLine();
2653}
2654
2655bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2656 llvm::GlobalVariable *Addr,
2657 bool PerformInit) {
2658 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2659 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2660 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link)
2661 return CGM.getLangOpts().OpenMPIsDevice;
2662 VD = VD->getDefinition(CGM.getContext());
2663 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2664 return CGM.getLangOpts().OpenMPIsDevice;
2665
2666 QualType ASTTy = VD->getType();
2667
2668 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2669 // Produce the unique prefix to identify the new target regions. We use
2670 // the source location of the variable declaration which we know to not
2671 // conflict with any target region.
2672 unsigned DeviceID;
2673 unsigned FileID;
2674 unsigned Line;
2675 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2676 SmallString<128> Buffer, Out;
2677 {
2678 llvm::raw_svector_ostream OS(Buffer);
2679 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2680 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2681 }
2682
2683 const Expr *Init = VD->getAnyInitializer();
2684 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2685 llvm::Constant *Ctor;
2686 llvm::Constant *ID;
2687 if (CGM.getLangOpts().OpenMPIsDevice) {
2688 // Generate function that re-emits the declaration's initializer into
2689 // the threadprivate copy of the variable VD
2690 CodeGenFunction CtorCGF(CGM);
2691
2692 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2693 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2694 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2695 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2696 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2697 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2698 FunctionArgList(), Loc, Loc);
2699 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2700 CtorCGF.EmitAnyExprToMem(Init,
2701 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2702 Init->getType().getQualifiers(),
2703 /*IsInitializer=*/true);
2704 CtorCGF.FinishFunction();
2705 Ctor = Fn;
2706 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2707 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2708 } else {
2709 Ctor = new llvm::GlobalVariable(
2710 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2711 llvm::GlobalValue::PrivateLinkage,
2712 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2713 ID = Ctor;
2714 }
2715
2716 // Register the information for the entry associated with the constructor.
2717 Out.clear();
2718 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2719 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2720 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2721 }
2722 if (VD->getType().isDestructedType() != QualType::DK_none) {
2723 llvm::Constant *Dtor;
2724 llvm::Constant *ID;
2725 if (CGM.getLangOpts().OpenMPIsDevice) {
2726 // Generate function that emits destructor call for the threadprivate
2727 // copy of the variable VD
2728 CodeGenFunction DtorCGF(CGM);
2729
2730 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2731 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2732 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2733 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2734 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2735 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2736 FunctionArgList(), Loc, Loc);
2737 // Create a scope with an artificial location for the body of this
2738 // function.
2739 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2740 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2741 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2742 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2743 DtorCGF.FinishFunction();
2744 Dtor = Fn;
2745 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2746 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2747 } else {
2748 Dtor = new llvm::GlobalVariable(
2749 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2750 llvm::GlobalValue::PrivateLinkage,
2751 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2752 ID = Dtor;
2753 }
2754 // Register the information for the entry associated with the destructor.
2755 Out.clear();
2756 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2757 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2758 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2759 }
2760 return CGM.getLangOpts().OpenMPIsDevice;
2761}
2762
2763Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2764 QualType VarType,
2765 StringRef Name) {
2766 std::string Suffix = getName({"artificial", ""});
2767 std::string CacheSuffix = getName({"cache", ""});
2768 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2769 llvm::Value *GAddr =
2770 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2771 llvm::Value *Args[] = {
2772 emitUpdateLocation(CGF, SourceLocation()),
2773 getThreadID(CGF, SourceLocation()),
2774 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2775 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2776 /*IsSigned=*/false),
2777 getOrCreateInternalVariable(
2778 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2779 return Address(
2780 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2781 CGF.EmitRuntimeCall(
2782 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2783 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2784 CGM.getPointerAlign());
2785}
2786
2787void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
2788 const RegionCodeGenTy &ThenGen,
2789 const RegionCodeGenTy &ElseGen) {
2790 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2791
2792 // If the condition constant folds and can be elided, try to avoid emitting
2793 // the condition and the dead arm of the if/else.
2794 bool CondConstant;
2795 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2796 if (CondConstant)
2797 ThenGen(CGF);
2798 else
2799 ElseGen(CGF);
2800 return;
2801 }
2802
2803 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2804 // emit the conditional branch.
2805 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2806 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2807 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2808 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2809
2810 // Emit the 'then' code.
2811 CGF.EmitBlock(ThenBlock);
2812 ThenGen(CGF);
2813 CGF.EmitBranch(ContBlock);
2814 // Emit the 'else' code if present.
2815 // There is no need to emit line number for unconditional branch.
2816 (void)ApplyDebugLocation::CreateEmpty(CGF);
2817 CGF.EmitBlock(ElseBlock);
2818 ElseGen(CGF);
2819 // There is no need to emit line number for unconditional branch.
2820 (void)ApplyDebugLocation::CreateEmpty(CGF);
2821 CGF.EmitBranch(ContBlock);
2822 // Emit the continuation block for code after the if.
2823 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2824}
2825
2826void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2827 llvm::Value *OutlinedFn,
2828 ArrayRef<llvm::Value *> CapturedVars,
2829 const Expr *IfCond) {
2830 if (!CGF.HaveInsertPoint())
2831 return;
2832 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2833 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
2834 PrePostActionTy &) {
2835 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2836 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2837 llvm::Value *Args[] = {
2838 RTLoc,
2839 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2840 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2841 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2842 RealArgs.append(std::begin(Args), std::end(Args));
2843 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2844
2845 llvm::Value *RTLFn = RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
2846 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2847 };
2848 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
2849 PrePostActionTy &) {
2850 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2851 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2852 // Build calls:
2853 // __kmpc_serialized_parallel(&Loc, GTid);
2854 llvm::Value *Args[] = {RTLoc, ThreadID};
2855 CGF.EmitRuntimeCall(
2856 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
2857
2858 // OutlinedFn(&GTid, &zero, CapturedStruct);
2859 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2860 /*Name*/ ".zero.addr");
2861 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2862 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2863 // ThreadId for serialized parallels is 0.
2864 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2865 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2866 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2867 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2868
2869 // __kmpc_end_serialized_parallel(&Loc, GTid);
2870 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2871 CGF.EmitRuntimeCall(
2872 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
2873 EndArgs);
2874 };
2875 if (IfCond) {
2876 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
2877 } else {
2878 RegionCodeGenTy ThenRCG(ThenGen);
2879 ThenRCG(CGF);
2880 }
2881}
2882
2883// If we're inside an (outlined) parallel region, use the region info's
2884// thread-ID variable (it is passed in a first argument of the outlined function
2885// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2886// regular serial code region, get thread ID by calling kmp_int32
2887// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2888// return the address of that temp.
2889Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2890 SourceLocation Loc) {
2891 if (auto *OMPRegionInfo =
2892 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2893 if (OMPRegionInfo->getThreadIDVariable())
2894 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2895
2896 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2897 QualType Int32Ty =
2898 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2899 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2900 CGF.EmitStoreOfScalar(ThreadID,
2901 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2902
2903 return ThreadIDTemp;
2904}
2905
2906llvm::Constant *
2907CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty,
2908 const llvm::Twine &Name) {
2909 SmallString<256> Buffer;
2910 llvm::raw_svector_ostream Out(Buffer);
2911 Out << Name;
2912 StringRef RuntimeName = Out.str();
2913 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2914 if (Elem.second) {
2915 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2916, __PRETTY_FUNCTION__))
2916 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2916, __PRETTY_FUNCTION__))
;
2917 return &*Elem.second;
2918 }
2919
2920 return Elem.second = new llvm::GlobalVariable(
2921 CGM.getModule(), Ty, /*IsConstant*/ false,
2922 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2923 Elem.first());
2924}
2925
2926llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2927 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2928 std::string Name = getName({Prefix, "var"});
2929 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2930}
2931
2932namespace {
2933/// Common pre(post)-action for different OpenMP constructs.
2934class CommonActionTy final : public PrePostActionTy {
2935 llvm::Value *EnterCallee;
2936 ArrayRef<llvm::Value *> EnterArgs;
2937 llvm::Value *ExitCallee;
2938 ArrayRef<llvm::Value *> ExitArgs;
2939 bool Conditional;
2940 llvm::BasicBlock *ContBlock = nullptr;
2941
2942public:
2943 CommonActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
2944 llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
2945 bool Conditional = false)
2946 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2947 ExitArgs(ExitArgs), Conditional(Conditional) {}
2948 void Enter(CodeGenFunction &CGF) override {
2949 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2950 if (Conditional) {
2951 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2952 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2953 ContBlock = CGF.createBasicBlock("omp_if.end");
2954 // Generate the branch (If-stmt)
2955 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2956 CGF.EmitBlock(ThenBlock);
2957 }
2958 }
2959 void Done(CodeGenFunction &CGF) {
2960 // Emit the rest of blocks/branches
2961 CGF.EmitBranch(ContBlock);
2962 CGF.EmitBlock(ContBlock, true);
2963 }
2964 void Exit(CodeGenFunction &CGF) override {
2965 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2966 }
2967};
2968} // anonymous namespace
2969
2970void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2971 StringRef CriticalName,
2972 const RegionCodeGenTy &CriticalOpGen,
2973 SourceLocation Loc, const Expr *Hint) {
2974 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2975 // CriticalOpGen();
2976 // __kmpc_end_critical(ident_t *, gtid, Lock);
2977 // Prepare arguments and build a call to __kmpc_critical
2978 if (!CGF.HaveInsertPoint())
2979 return;
2980 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2981 getCriticalRegionLock(CriticalName)};
2982 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2983 std::end(Args));
2984 if (Hint) {
2985 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2986 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
2987 }
2988 CommonActionTy Action(
2989 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
2990 : OMPRTL__kmpc_critical),
2991 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
2992 CriticalOpGen.setAction(Action);
2993 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2994}
2995
2996void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2997 const RegionCodeGenTy &MasterOpGen,
2998 SourceLocation Loc) {
2999 if (!CGF.HaveInsertPoint())
3000 return;
3001 // if(__kmpc_master(ident_t *, gtid)) {
3002 // MasterOpGen();
3003 // __kmpc_end_master(ident_t *, gtid);
3004 // }
3005 // Prepare arguments and build a call to __kmpc_master
3006 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3007 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3008 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3009 /*Conditional=*/true);
3010 MasterOpGen.setAction(Action);
3011 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3012 Action.Done(CGF);
3013}
3014
3015void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3016 SourceLocation Loc) {
3017 if (!CGF.HaveInsertPoint())
3018 return;
3019 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3020 llvm::Value *Args[] = {
3021 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3022 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3023 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3024 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3025 Region->emitUntiedSwitch(CGF);
3026}
3027
3028void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3029 const RegionCodeGenTy &TaskgroupOpGen,
3030 SourceLocation Loc) {
3031 if (!CGF.HaveInsertPoint())
3032 return;
3033 // __kmpc_taskgroup(ident_t *, gtid);
3034 // TaskgroupOpGen();
3035 // __kmpc_end_taskgroup(ident_t *, gtid);
3036 // Prepare arguments and build a call to __kmpc_taskgroup
3037 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3038 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3039 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3040 Args);
3041 TaskgroupOpGen.setAction(Action);
3042 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3043}
3044
3045/// Given an array of pointers to variables, project the address of a
3046/// given variable.
3047static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3048 unsigned Index, const VarDecl *Var) {
3049 // Pull out the pointer to the variable.
3050 Address PtrAddr =
3051 CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize());
3052 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3053
3054 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3055 Addr = CGF.Builder.CreateElementBitCast(
3056 Addr, CGF.ConvertTypeForMem(Var->getType()));
3057 return Addr;
3058}
3059
3060static llvm::Value *emitCopyprivateCopyFunction(
3061 CodeGenModule &CGM, llvm::Type *ArgsType,
3062 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3063 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3064 SourceLocation Loc) {
3065 ASTContext &C = CGM.getContext();
3066 // void copy_func(void *LHSArg, void *RHSArg);
3067 FunctionArgList Args;
3068 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3069 ImplicitParamDecl::Other);
3070 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3071 ImplicitParamDecl::Other);
3072 Args.push_back(&LHSArg);
3073 Args.push_back(&RHSArg);
3074 const auto &CGFI =
3075 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3076 std::string Name =
3077 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3078 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3079 llvm::GlobalValue::InternalLinkage, Name,
3080 &CGM.getModule());
3081 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3082 Fn->setDoesNotRecurse();
3083 CodeGenFunction CGF(CGM);
3084 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3085 // Dest = (void*[n])(LHSArg);
3086 // Src = (void*[n])(RHSArg);
3087 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3088 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3089 ArgsType), CGF.getPointerAlign());
3090 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3091 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3092 ArgsType), CGF.getPointerAlign());
3093 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3094 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3095 // ...
3096 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3097 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3098 const auto *DestVar =
3099 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3100 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3101
3102 const auto *SrcVar =
3103 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3104 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3105
3106 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3107 QualType Type = VD->getType();
3108 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3109 }
3110 CGF.FinishFunction();
3111 return Fn;
3112}
3113
3114void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3115 const RegionCodeGenTy &SingleOpGen,
3116 SourceLocation Loc,
3117 ArrayRef<const Expr *> CopyprivateVars,
3118 ArrayRef<const Expr *> SrcExprs,
3119 ArrayRef<const Expr *> DstExprs,
3120 ArrayRef<const Expr *> AssignmentOps) {
3121 if (!CGF.HaveInsertPoint())
3122 return;
3123 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3125, __PRETTY_FUNCTION__))
3124 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3125, __PRETTY_FUNCTION__))
3125 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3125, __PRETTY_FUNCTION__))
;
3126 ASTContext &C = CGM.getContext();
3127 // int32 did_it = 0;
3128 // if(__kmpc_single(ident_t *, gtid)) {
3129 // SingleOpGen();
3130 // __kmpc_end_single(ident_t *, gtid);
3131 // did_it = 1;
3132 // }
3133 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3134 // <copy_func>, did_it);
3135
3136 Address DidIt = Address::invalid();
3137 if (!CopyprivateVars.empty()) {
3138 // int32 did_it = 0;
3139 QualType KmpInt32Ty =
3140 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3141 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3142 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3143 }
3144 // Prepare arguments and build a call to __kmpc_single
3145 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3146 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3147 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3148 /*Conditional=*/true);
3149 SingleOpGen.setAction(Action);
3150 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3151 if (DidIt.isValid()) {
3152 // did_it = 1;
3153 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3154 }
3155 Action.Done(CGF);
3156 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3157 // <copy_func>, did_it);
3158 if (DidIt.isValid()) {
3159 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3160 QualType CopyprivateArrayTy =
3161 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3162 /*IndexTypeQuals=*/0);
3163 // Create a list of all private variables for copyprivate.
3164 Address CopyprivateList =
3165 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3166 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3167 Address Elem = CGF.Builder.CreateConstArrayGEP(
3168 CopyprivateList, I, CGF.getPointerSize());
3169 CGF.Builder.CreateStore(
3170 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3171 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3172 Elem);
3173 }
3174 // Build function that copies private values from single region to all other
3175 // threads in the corresponding parallel region.
3176 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3177 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3178 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3179 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3180 Address CL =
3181 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3182 CGF.VoidPtrTy);
3183 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3184 llvm::Value *Args[] = {
3185 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3186 getThreadID(CGF, Loc), // i32 <gtid>
3187 BufSize, // size_t <buf_size>
3188 CL.getPointer(), // void *<copyprivate list>
3189 CpyFn, // void (*) (void *, void *) <copy_func>
3190 DidItVal // i32 did_it
3191 };
3192 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3193 }
3194}
3195
3196void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3197 const RegionCodeGenTy &OrderedOpGen,
3198 SourceLocation Loc, bool IsThreads) {
3199 if (!CGF.HaveInsertPoint())
3200 return;
3201 // __kmpc_ordered(ident_t *, gtid);
3202 // OrderedOpGen();
3203 // __kmpc_end_ordered(ident_t *, gtid);
3204 // Prepare arguments and build a call to __kmpc_ordered
3205 if (IsThreads) {
3206 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3207 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3208 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3209 Args);
3210 OrderedOpGen.setAction(Action);
3211 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3212 return;
3213 }
3214 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3215}
3216
3217unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3218 unsigned Flags;
3219 if (Kind == OMPD_for)
3220 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3221 else if (Kind == OMPD_sections)
3222 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3223 else if (Kind == OMPD_single)
3224 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3225 else if (Kind == OMPD_barrier)
3226 Flags = OMP_IDENT_BARRIER_EXPL;
3227 else
3228 Flags = OMP_IDENT_BARRIER_IMPL;
3229 return Flags;
3230}
3231
3232void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3233 OpenMPDirectiveKind Kind, bool EmitChecks,
3234 bool ForceSimpleCall) {
3235 if (!CGF.HaveInsertPoint())
3236 return;
3237 // Build call __kmpc_cancel_barrier(loc, thread_id);
3238 // Build call __kmpc_barrier(loc, thread_id);
3239 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3240 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3241 // thread_id);
3242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3243 getThreadID(CGF, Loc)};
3244 if (auto *OMPRegionInfo =
3245 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3246 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3247 llvm::Value *Result = CGF.EmitRuntimeCall(
3248 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3249 if (EmitChecks) {
3250 // if (__kmpc_cancel_barrier()) {
3251 // exit from construct;
3252 // }
3253 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3254 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3255 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3256 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3257 CGF.EmitBlock(ExitBB);
3258 // exit from construct;
3259 CodeGenFunction::JumpDest CancelDestination =
3260 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3261 CGF.EmitBranchThroughCleanup(CancelDestination);
3262 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3263 }
3264 return;
3265 }
3266 }
3267 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3268}
3269
3270/// Map the OpenMP loop schedule to the runtime enumeration.
3271static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3272 bool Chunked, bool Ordered) {
3273 switch (ScheduleKind) {
3274 case OMPC_SCHEDULE_static:
3275 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3276 : (Ordered ? OMP_ord_static : OMP_sch_static);
3277 case OMPC_SCHEDULE_dynamic:
3278 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3279 case OMPC_SCHEDULE_guided:
3280 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3281 case OMPC_SCHEDULE_runtime:
3282 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3283 case OMPC_SCHEDULE_auto:
3284 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3285 case OMPC_SCHEDULE_unknown:
3286 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3286, __PRETTY_FUNCTION__))
;
3287 return Ordered ? OMP_ord_static : OMP_sch_static;
3288 }
3289 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3289)
;
3290}
3291
3292/// Map the OpenMP distribute schedule to the runtime enumeration.
3293static OpenMPSchedType
3294getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3295 // only static is allowed for dist_schedule
3296 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3297}
3298
3299bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3300 bool Chunked) const {
3301 OpenMPSchedType Schedule =
3302 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3303 return Schedule == OMP_sch_static;
3304}
3305
3306bool CGOpenMPRuntime::isStaticNonchunked(
3307 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3308 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3309 return Schedule == OMP_dist_sch_static;
3310}
3311
3312bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3313 bool Chunked) const {
3314 OpenMPSchedType Schedule =
3315 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3316 return Schedule == OMP_sch_static_chunked;
3317}
3318
3319bool CGOpenMPRuntime::isStaticChunked(
3320 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3321 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3322 return Schedule == OMP_dist_sch_static_chunked;
3323}
3324
3325bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3326 OpenMPSchedType Schedule =
3327 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3328 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3328, __PRETTY_FUNCTION__))
;
3329 return Schedule != OMP_sch_static;
3330}
3331
3332static int addMonoNonMonoModifier(OpenMPSchedType Schedule,
3333 OpenMPScheduleClauseModifier M1,
3334 OpenMPScheduleClauseModifier M2) {
3335 int Modifier = 0;
3336 switch (M1) {
3337 case OMPC_SCHEDULE_MODIFIER_monotonic:
3338 Modifier = OMP_sch_modifier_monotonic;
3339 break;
3340 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3341 Modifier = OMP_sch_modifier_nonmonotonic;
3342 break;
3343 case OMPC_SCHEDULE_MODIFIER_simd:
3344 if (Schedule == OMP_sch_static_chunked)
3345 Schedule = OMP_sch_static_balanced_chunked;
3346 break;
3347 case OMPC_SCHEDULE_MODIFIER_last:
3348 case OMPC_SCHEDULE_MODIFIER_unknown:
3349 break;
3350 }
3351 switch (M2) {
3352 case OMPC_SCHEDULE_MODIFIER_monotonic:
3353 Modifier = OMP_sch_modifier_monotonic;
3354 break;
3355 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3356 Modifier = OMP_sch_modifier_nonmonotonic;
3357 break;
3358 case OMPC_SCHEDULE_MODIFIER_simd:
3359 if (Schedule == OMP_sch_static_chunked)
3360 Schedule = OMP_sch_static_balanced_chunked;
3361 break;
3362 case OMPC_SCHEDULE_MODIFIER_last:
3363 case OMPC_SCHEDULE_MODIFIER_unknown:
3364 break;
3365 }
3366 return Schedule | Modifier;
3367}
3368
3369void CGOpenMPRuntime::emitForDispatchInit(
3370 CodeGenFunction &CGF, SourceLocation Loc,
3371 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3372 bool Ordered, const DispatchRTInput &DispatchValues) {
3373 if (!CGF.HaveInsertPoint())
3374 return;
3375 OpenMPSchedType Schedule = getRuntimeSchedule(
3376 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3377 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3380, __PRETTY_FUNCTION__))
3378 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3380, __PRETTY_FUNCTION__))
3379 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3380, __PRETTY_FUNCTION__))
3380 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3380, __PRETTY_FUNCTION__))
;
3381 // Call __kmpc_dispatch_init(
3382 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3383 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3384 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3385
3386 // If the Chunk was not specified in the clause - use default value 1.
3387 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3388 : CGF.Builder.getIntN(IVSize, 1);
3389 llvm::Value *Args[] = {
3390 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3391 CGF.Builder.getInt32(addMonoNonMonoModifier(
3392 Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3393 DispatchValues.LB, // Lower
3394 DispatchValues.UB, // Upper
3395 CGF.Builder.getIntN(IVSize, 1), // Stride
3396 Chunk // Chunk
3397 };
3398 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3399}
3400
3401static void emitForStaticInitCall(
3402 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3403 llvm::Constant *ForStaticInitFunction, OpenMPSchedType Schedule,
3404 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3405 const CGOpenMPRuntime::StaticRTInput &Values) {
3406 if (!CGF.HaveInsertPoint())
3407 return;
3408
3409 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3409, __PRETTY_FUNCTION__))
;
3410 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3414, __PRETTY_FUNCTION__))
3411 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3414, __PRETTY_FUNCTION__))
3412 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3414, __PRETTY_FUNCTION__))
3413 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3414, __PRETTY_FUNCTION__))
3414 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3414, __PRETTY_FUNCTION__))
;
3415
3416 // Call __kmpc_for_static_init(
3417 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3418 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3419 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3420 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3421 llvm::Value *Chunk = Values.Chunk;
3422 if (Chunk == nullptr) {
3423 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3425, __PRETTY_FUNCTION__))
3424 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3425, __PRETTY_FUNCTION__))
3425 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3425, __PRETTY_FUNCTION__))
;
3426 // If the Chunk was not specified in the clause - use default value 1.
3427 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3428 } else {
3429 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3433, __PRETTY_FUNCTION__))
3430 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3433, __PRETTY_FUNCTION__))
3431 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3433, __PRETTY_FUNCTION__))
3432 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3433, __PRETTY_FUNCTION__))
3433 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3433, __PRETTY_FUNCTION__))
;
3434 }
3435 llvm::Value *Args[] = {
3436 UpdateLocation,
3437 ThreadId,
3438 CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1,
3439 M2)), // Schedule type
3440 Values.IL.getPointer(), // &isLastIter
3441 Values.LB.getPointer(), // &LB
3442 Values.UB.getPointer(), // &UB
3443 Values.ST.getPointer(), // &Stride
3444 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3445 Chunk // Chunk
3446 };
3447 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3448}
3449
3450void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3451 SourceLocation Loc,
3452 OpenMPDirectiveKind DKind,
3453 const OpenMPScheduleTy &ScheduleKind,
3454 const StaticRTInput &Values) {
3455 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3456 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3457 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3458, __PRETTY_FUNCTION__))
3458 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3458, __PRETTY_FUNCTION__))
;
3459 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3460 isOpenMPLoopDirective(DKind)
3461 ? OMP_IDENT_WORK_LOOP
3462 : OMP_IDENT_WORK_SECTIONS);
3463 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3464 llvm::Constant *StaticInitFunction =
3465 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3466 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3467 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3468}
3469
3470void CGOpenMPRuntime::emitDistributeStaticInit(
3471 CodeGenFunction &CGF, SourceLocation Loc,
3472 OpenMPDistScheduleClauseKind SchedKind,
3473 const CGOpenMPRuntime::StaticRTInput &Values) {
3474 OpenMPSchedType ScheduleNum =
3475 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3476 llvm::Value *UpdatedLocation =
3477 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3478 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3479 llvm::Constant *StaticInitFunction =
3480 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3481 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3482 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3483 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3484}
3485
3486void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3487 SourceLocation Loc,
3488 OpenMPDirectiveKind DKind) {
3489 if (!CGF.HaveInsertPoint())
3490 return;
3491 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3492 llvm::Value *Args[] = {
3493 emitUpdateLocation(CGF, Loc,
3494 isOpenMPDistributeDirective(DKind)
3495 ? OMP_IDENT_WORK_DISTRIBUTE
3496 : isOpenMPLoopDirective(DKind)
3497 ? OMP_IDENT_WORK_LOOP
3498 : OMP_IDENT_WORK_SECTIONS),
3499 getThreadID(CGF, Loc)};
3500 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3501 Args);
3502}
3503
3504void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3505 SourceLocation Loc,
3506 unsigned IVSize,
3507 bool IVSigned) {
3508 if (!CGF.HaveInsertPoint())
3509 return;
3510 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3511 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3512 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3513}
3514
3515llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3516 SourceLocation Loc, unsigned IVSize,
3517 bool IVSigned, Address IL,
3518 Address LB, Address UB,
3519 Address ST) {
3520 // Call __kmpc_dispatch_next(
3521 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3522 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3523 // kmp_int[32|64] *p_stride);
3524 llvm::Value *Args[] = {
3525 emitUpdateLocation(CGF, Loc),
3526 getThreadID(CGF, Loc),
3527 IL.getPointer(), // &isLastIter
3528 LB.getPointer(), // &Lower
3529 UB.getPointer(), // &Upper
3530 ST.getPointer() // &Stride
3531 };
3532 llvm::Value *Call =
3533 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3534 return CGF.EmitScalarConversion(
3535 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3536 CGF.getContext().BoolTy, Loc);
3537}
3538
3539void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3540 llvm::Value *NumThreads,
3541 SourceLocation Loc) {
3542 if (!CGF.HaveInsertPoint())
3543 return;
3544 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3545 llvm::Value *Args[] = {
3546 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3547 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3548 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3549 Args);
3550}
3551
3552void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3553 OpenMPProcBindClauseKind ProcBind,
3554 SourceLocation Loc) {
3555 if (!CGF.HaveInsertPoint())
3556 return;
3557 // Constants for proc bind value accepted by the runtime.
3558 enum ProcBindTy {
3559 ProcBindFalse = 0,
3560 ProcBindTrue,
3561 ProcBindMaster,
3562 ProcBindClose,
3563 ProcBindSpread,
3564 ProcBindIntel,
3565 ProcBindDefault
3566 } RuntimeProcBind;
3567 switch (ProcBind) {
3568 case OMPC_PROC_BIND_master:
3569 RuntimeProcBind = ProcBindMaster;
3570 break;
3571 case OMPC_PROC_BIND_close:
3572 RuntimeProcBind = ProcBindClose;
3573 break;
3574 case OMPC_PROC_BIND_spread:
3575 RuntimeProcBind = ProcBindSpread;
3576 break;
3577 case OMPC_PROC_BIND_unknown:
3578 llvm_unreachable("Unsupported proc_bind value.")::llvm::llvm_unreachable_internal("Unsupported proc_bind value."
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3578)
;
3579 }
3580 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3581 llvm::Value *Args[] = {
3582 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3583 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3584 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3585}
3586
3587void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3588 SourceLocation Loc) {
3589 if (!CGF.HaveInsertPoint())
3590 return;
3591 // Build call void __kmpc_flush(ident_t *loc)
3592 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3593 emitUpdateLocation(CGF, Loc));
3594}
3595
3596namespace {
3597/// Indexes of fields for type kmp_task_t.
3598enum KmpTaskTFields {
3599 /// List of shared variables.
3600 KmpTaskTShareds,
3601 /// Task routine.
3602 KmpTaskTRoutine,
3603 /// Partition id for the untied tasks.
3604 KmpTaskTPartId,
3605 /// Function with call of destructors for private variables.
3606 Data1,
3607 /// Task priority.
3608 Data2,
3609 /// (Taskloops only) Lower bound.
3610 KmpTaskTLowerBound,
3611 /// (Taskloops only) Upper bound.
3612 KmpTaskTUpperBound,
3613 /// (Taskloops only) Stride.
3614 KmpTaskTStride,
3615 /// (Taskloops only) Is last iteration flag.
3616 KmpTaskTLastIter,
3617 /// (Taskloops only) Reduction data.
3618 KmpTaskTReductions,
3619};
3620} // anonymous namespace
3621
3622bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3623 return OffloadEntriesTargetRegion.empty() &&
3624 OffloadEntriesDeviceGlobalVar.empty();
3625}
3626
3627/// Initialize target region entry.
3628void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3629 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3630 StringRef ParentName, unsigned LineNum,
3631 unsigned Order) {
3632 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3634, __PRETTY_FUNCTION__))
3633 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3634, __PRETTY_FUNCTION__))
3634 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3634, __PRETTY_FUNCTION__))
;
3635 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3636 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3637 OMPTargetRegionEntryTargetRegion);
3638 ++OffloadingEntriesNum;
3639}
3640
3641void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3642 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3643 StringRef ParentName, unsigned LineNum,
3644 llvm::Constant *Addr, llvm::Constant *ID,
3645 OMPTargetRegionEntryKind Flags) {
3646 // If we are emitting code for a target, the entry is already initialized,
3647 // only has to be registered.
3648 if (CGM.getLangOpts().OpenMPIsDevice) {
3649 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3650 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3651 DiagnosticsEngine::Error,
3652 "Unable to find target region on line '%0' in the device code.");
3653 CGM.getDiags().Report(DiagID) << LineNum;
3654 return;
3655 }
3656 auto &Entry =
3657 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3658 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3658, __PRETTY_FUNCTION__))
;
3659 Entry.setAddress(Addr);
3660 Entry.setID(ID);
3661 Entry.setFlags(Flags);
3662 } else {
3663 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3664 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3665 ++OffloadingEntriesNum;
3666 }
3667}
3668
3669bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3670 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3671 unsigned LineNum) const {
3672 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3673 if (PerDevice == OffloadEntriesTargetRegion.end())
3674 return false;
3675 auto PerFile = PerDevice->second.find(FileID);
3676 if (PerFile == PerDevice->second.end())
3677 return false;
3678 auto PerParentName = PerFile->second.find(ParentName);
3679 if (PerParentName == PerFile->second.end())
3680 return false;
3681 auto PerLine = PerParentName->second.find(LineNum);
3682 if (PerLine == PerParentName->second.end())
3683 return false;
3684 // Fail if this entry is already registered.
3685 if (PerLine->second.getAddress() || PerLine->second.getID())
3686 return false;
3687 return true;
3688}
3689
3690void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3691 const OffloadTargetRegionEntryInfoActTy &Action) {
3692 // Scan all target region entries and perform the provided action.
3693 for (const auto &D : OffloadEntriesTargetRegion)
3694 for (const auto &F : D.second)
3695 for (const auto &P : F.second)
3696 for (const auto &L : P.second)
3697 Action(D.first, F.first, P.first(), L.first, L.second);
3698}
3699
3700void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3701 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3702 OMPTargetGlobalVarEntryKind Flags,
3703 unsigned Order) {
3704 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3705 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3706 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3708 ++OffloadingEntriesNum;
3709}
3710
3711void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3712 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3713 CharUnits VarSize,
3714 OMPTargetGlobalVarEntryKind Flags,
3715 llvm::GlobalValue::LinkageTypes Linkage) {
3716 if (CGM.getLangOpts().OpenMPIsDevice) {
3717 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3718 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3719, __PRETTY_FUNCTION__))
3719 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3719, __PRETTY_FUNCTION__))
;
3720 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3721, __PRETTY_FUNCTION__))
3721 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3721, __PRETTY_FUNCTION__))
;
3722 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName))
3723 return;
3724 Entry.setAddress(Addr);
3725 Entry.setVarSize(VarSize);
3726 Entry.setLinkage(Linkage);
3727 } else {
3728 if (hasDeviceGlobalVarEntryInfo(VarName))
3729 return;
3730 OffloadEntriesDeviceGlobalVar.try_emplace(
3731 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3732 ++OffloadingEntriesNum;
3733 }
3734}
3735
3736void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3737 actOnDeviceGlobalVarEntriesInfo(
3738 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3739 // Scan all target region entries and perform the provided action.
3740 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3741 Action(E.getKey(), E.getValue());
3742}
3743
3744llvm::Function *
3745CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
3746 // If we don't have entries or if we are emitting code for the device, we
3747 // don't need to do anything.
3748 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
3749 return nullptr;
3750
3751 llvm::Module &M = CGM.getModule();
3752 ASTContext &C = CGM.getContext();
3753
3754 // Get list of devices we care about
3755 const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
3756
3757 // We should be creating an offloading descriptor only if there are devices
3758 // specified.
3759 assert(!Devices.empty() && "No OpenMP offloading devices??")((!Devices.empty() && "No OpenMP offloading devices??"
) ? static_cast<void> (0) : __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3759, __PRETTY_FUNCTION__))
;
3760
3761 // Create the external variables that will point to the begin and end of the
3762 // host entries section. These will be defined by the linker.
3763 llvm::Type *OffloadEntryTy =
3764 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
3765 std::string EntriesBeginName = getName({"omp_offloading", "entries_begin"});
3766 auto *HostEntriesBegin = new llvm::GlobalVariable(
3767 M, OffloadEntryTy, /*isConstant=*/true,
3768 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
3769 EntriesBeginName);
3770 std::string EntriesEndName = getName({"omp_offloading", "entries_end"});
3771 auto *HostEntriesEnd =
3772 new llvm::GlobalVariable(M, OffloadEntryTy, /*isConstant=*/true,
3773 llvm::GlobalValue::ExternalLinkage,
3774 /*Initializer=*/nullptr, EntriesEndName);
3775
3776 // Create all device images
3777 auto *DeviceImageTy = cast<llvm::StructType>(
3778 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
3779 ConstantInitBuilder DeviceImagesBuilder(CGM);
3780 ConstantArrayBuilder DeviceImagesEntries =
3781 DeviceImagesBuilder.beginArray(DeviceImageTy);
3782
3783 for (const llvm::Triple &Device : Devices) {
3784 StringRef T = Device.getTriple();
3785 std::string BeginName = getName({"omp_offloading", "img_start", ""});
3786 auto *ImgBegin = new llvm::GlobalVariable(
3787 M, CGM.Int8Ty, /*isConstant=*/true,
3788 llvm::GlobalValue::ExternalWeakLinkage,
3789 /*Initializer=*/nullptr, Twine(BeginName).concat(T));
3790 std::string EndName = getName({"omp_offloading", "img_end", ""});
3791 auto *ImgEnd = new llvm::GlobalVariable(
3792 M, CGM.Int8Ty, /*isConstant=*/true,
3793 llvm::GlobalValue::ExternalWeakLinkage,
3794 /*Initializer=*/nullptr, Twine(EndName).concat(T));
3795
3796 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
3797 HostEntriesEnd};
3798 createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
3799 DeviceImagesEntries);
3800 }
3801
3802 // Create device images global array.
3803 std::string ImagesName = getName({"omp_offloading", "device_images"});
3804 llvm::GlobalVariable *DeviceImages =
3805 DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
3806 CGM.getPointerAlign(),
3807 /*isConstant=*/true);
3808 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3809
3810 // This is a Zero array to be used in the creation of the constant expressions
3811 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3812 llvm::Constant::getNullValue(CGM.Int32Ty)};
3813
3814 // Create the target region descriptor.
3815 llvm::Constant *Data[] = {
3816 llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
3817 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3818 DeviceImages, Index),
3819 HostEntriesBegin, HostEntriesEnd};
3820 std::string Descriptor = getName({"omp_offloading", "descriptor"});
3821 llvm::GlobalVariable *Desc = createGlobalStruct(
3822 CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
3823
3824 // Emit code to register or unregister the descriptor at execution
3825 // startup or closing, respectively.
3826
3827 llvm::Function *UnRegFn;
3828 {
3829 FunctionArgList Args;
3830 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
3831 Args.push_back(&DummyPtr);
3832
3833 CodeGenFunction CGF(CGM);
3834 // Disable debug info for global (de-)initializer because they are not part
3835 // of some particular construct.
3836 CGF.disableDebugInfo();
3837 const auto &FI =
3838 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3839 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3840 std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
3841 UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
3842 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
3843 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
3844 Desc);
3845 CGF.FinishFunction();
3846 }
3847 llvm::Function *RegFn;
3848 {
3849 CodeGenFunction CGF(CGM);
3850 // Disable debug info for global (de-)initializer because they are not part
3851 // of some particular construct.
3852 CGF.disableDebugInfo();
3853 const auto &FI = CGM.getTypes().arrangeNullaryFunction();
3854 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3855
3856 // Encode offload target triples into the registration function name. It
3857 // will serve as a comdat key for the registration/unregistration code for
3858 // this particular combination of offloading targets.
3859 SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
3860 RegFnNameParts[0] = "omp_offloading";
3861 RegFnNameParts[1] = "descriptor_reg";
3862 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
3863 [](const llvm::Triple &T) -> const std::string& {
3864 return T.getTriple();
3865 });
3866 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
3867 std::string Descriptor = getName(RegFnNameParts);
3868 RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
3869 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
3870 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
3871 // Create a variable to drive the registration and unregistration of the
3872 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
3873 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
3874 SourceLocation(), nullptr, C.CharTy,
3875 ImplicitParamDecl::Other);
3876 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3877 CGF.FinishFunction();
3878 }
3879 if (CGM.supportsCOMDAT()) {
3880 // It is sufficient to call registration function only once, so create a
3881 // COMDAT group for registration/unregistration functions and associated
3882 // data. That would reduce startup time and code size. Registration
3883 // function serves as a COMDAT group key.
3884 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
3885 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
3886 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3887 RegFn->setComdat(ComdatKey);
3888 UnRegFn->setComdat(ComdatKey);
3889 DeviceImages->setComdat(ComdatKey);
3890 Desc->setComdat(ComdatKey);
3891 }
3892 return RegFn;
3893}
3894
3895void CGOpenMPRuntime::createOffloadEntry(
3896 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3897 llvm::GlobalValue::LinkageTypes Linkage) {
3898 StringRef Name = Addr->getName();
3899 llvm::Module &M = CGM.getModule();
3900 llvm::LLVMContext &C = M.getContext();
3901
3902 // Create constant string with the name.
3903 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3904
3905 std::string StringName = getName({"omp_offloading", "entry_name"});
3906 auto *Str = new llvm::GlobalVariable(
3907 M, StrPtrInit->getType(), /*isConstant=*/true,
3908 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3909 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3910
3911 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
3912 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
3913 llvm::ConstantInt::get(CGM.SizeTy, Size),
3914 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3915 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3916 std::string EntryName = getName({"omp_offloading", "entry", ""});
3917 llvm::GlobalVariable *Entry = createGlobalStruct(
3918 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
3919 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
3920
3921 // The entry has to be created in the section the linker expects it to be.
3922 std::string Section = getName({"omp_offloading", "entries"});
3923 Entry->setSection(Section);
3924}
3925
3926void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3927 // Emit the offloading entries and metadata so that the device codegen side
3928 // can easily figure out what to emit. The produced metadata looks like
3929 // this:
3930 //
3931 // !omp_offload.info = !{!1, ...}
3932 //
3933 // Right now we only generate metadata for function that contain target
3934 // regions.
3935
3936 // If we do not have entries, we don't need to do anything.
3937 if (OffloadEntriesInfoManager.empty())
3938 return;
3939
3940 llvm::Module &M = CGM.getModule();
3941 llvm::LLVMContext &C = M.getContext();
3942 SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
3943 OrderedEntries(OffloadEntriesInfoManager.size());
3944 llvm::SmallVector<StringRef, 16> ParentFunctions(
3945 OffloadEntriesInfoManager.size());
3946
3947 // Auxiliary methods to create metadata values and strings.
3948 auto &&GetMDInt = [this](unsigned V) {
3949 return llvm::ConstantAsMetadata::get(
3950 llvm::ConstantInt::get(CGM.Int32Ty, V));
3951 };
3952
3953 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3954
3955 // Create the offloading info metadata node.
3956 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3957
3958 // Create function that emits metadata for each target region entry;
3959 auto &&TargetRegionMetadataEmitter =
3960 [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
3961 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3962 unsigned Line,
3963 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3964 // Generate metadata for target regions. Each entry of this metadata
3965 // contains:
3966 // - Entry 0 -> Kind of this type of metadata (0).
3967 // - Entry 1 -> Device ID of the file where the entry was identified.
3968 // - Entry 2 -> File ID of the file where the entry was identified.
3969 // - Entry 3 -> Mangled name of the function where the entry was
3970 // identified.
3971 // - Entry 4 -> Line in the file where the entry was identified.
3972 // - Entry 5 -> Order the entry was created.
3973 // The first element of the metadata node is the kind.
3974 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3975 GetMDInt(FileID), GetMDString(ParentName),
3976 GetMDInt(Line), GetMDInt(E.getOrder())};
3977
3978 // Save this entry in the right position of the ordered entries array.
3979 OrderedEntries[E.getOrder()] = &E;
3980 ParentFunctions[E.getOrder()] = ParentName;
3981
3982 // Add metadata to the named metadata node.
3983 MD->addOperand(llvm::MDNode::get(C, Ops));
3984 };
3985
3986 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3987 TargetRegionMetadataEmitter);
3988
3989 // Create function that emits metadata for each device global variable entry;
3990 auto &&DeviceGlobalVarMetadataEmitter =
3991 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3992 MD](StringRef MangledName,
3993 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3994 &E) {
3995 // Generate metadata for global variables. Each entry of this metadata
3996 // contains:
3997 // - Entry 0 -> Kind of this type of metadata (1).
3998 // - Entry 1 -> Mangled name of the variable.
3999 // - Entry 2 -> Declare target kind.
4000 // - Entry 3 -> Order the entry was created.
4001 // The first element of the metadata node is the kind.
4002 llvm::Metadata *Ops[] = {
4003 GetMDInt(E.getKind()), GetMDString(MangledName),
4004 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4005
4006 // Save this entry in the right position of the ordered entries array.
4007 OrderedEntries[E.getOrder()] = &E;
4008
4009 // Add metadata to the named metadata node.
4010 MD->addOperand(llvm::MDNode::get(C, Ops));
4011 };
4012
4013 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4014 DeviceGlobalVarMetadataEmitter);
4015
4016 for (const auto *E : OrderedEntries) {
4017 assert(E && "All ordered entries must exist!")((E && "All ordered entries must exist!") ? static_cast
<void> (0) : __assert_fail ("E && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4017, __PRETTY_FUNCTION__))
;
4018 if (const auto *CE =
4019 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4020 E)) {
4021 if (!CE->getID() || !CE->getAddress()) {
4022 // Do not blame the entry if the parent funtion is not emitted.
4023 StringRef FnName = ParentFunctions[CE->getOrder()];
4024 if (!CGM.GetGlobalValue(FnName))
4025 continue;
4026 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4027 DiagnosticsEngine::Error,
4028 "Offloading entry for target region is incorrect: either the "
4029 "address or the ID is invalid.");
4030 CGM.getDiags().Report(DiagID);
4031 continue;
4032 }
4033 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4034 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4035 } else if (const auto *CE =
4036 dyn_cast<OffloadEntriesInfoManagerTy::
4037 OffloadEntryInfoDeviceGlobalVar>(E)) {
4038 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4039 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4040 CE->getFlags());
4041 switch (Flags) {
4042 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4043 if (!CE->getAddress()) {
4044 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4045 DiagnosticsEngine::Error,
4046 "Offloading entry for declare target variable is incorrect: the "
4047 "address is invalid.");
4048 CGM.getDiags().Report(DiagID);
4049 continue;
4050 }
4051 // The vaiable has no definition - no need to add the entry.
4052 if (CE->getVarSize().isZero())
4053 continue;
4054 break;
4055 }
4056 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4057 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4059, __PRETTY_FUNCTION__))
4058 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4059, __PRETTY_FUNCTION__))
4059 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4059, __PRETTY_FUNCTION__))
;
4060 if (CGM.getLangOpts().OpenMPIsDevice)
4061 continue;
4062 if (!CE->getAddress()) {
4063 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4064 DiagnosticsEngine::Error,
4065 "Offloading entry for declare target variable is incorrect: the "
4066 "address is invalid.");
4067 CGM.getDiags().Report(DiagID);
4068 continue;
4069 }
4070 break;
4071 }
4072 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4073 CE->getVarSize().getQuantity(), Flags,
4074 CE->getLinkage());
4075 } else {
4076 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4076)
;
4077 }
4078 }
4079}
4080
4081/// Loads all the offload entries information from the host IR
4082/// metadata.
4083void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4084 // If we are in target mode, load the metadata from the host IR. This code has
4085 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4086
4087 if (!CGM.getLangOpts().OpenMPIsDevice)
4088 return;
4089
4090 if (CGM.getLangOpts().OMPHostIRFile.empty())
4091 return;
4092
4093 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4094 if (auto EC = Buf.getError()) {
4095 CGM.getDiags().Report(diag::err_cannot_open_file)
4096 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4097 return;
4098 }
4099
4100 llvm::LLVMContext C;
4101 auto ME = expectedToErrorOrAndEmitErrors(
4102 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4103
4104 if (auto EC = ME.getError()) {
4105 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4106 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4107 CGM.getDiags().Report(DiagID)
4108 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4109 return;
4110 }
4111
4112 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4113 if (!MD)
4114 return;
4115
4116 for (llvm::MDNode *MN : MD->operands()) {
4117 auto &&GetMDInt = [MN](unsigned Idx) {
4118 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4119 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4120 };
4121
4122 auto &&GetMDString = [MN](unsigned Idx) {
4123 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4124 return V->getString();
4125 };
4126
4127 switch (GetMDInt(0)) {
4128 default:
4129 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4129)
;
4130 break;
4131 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4132 OffloadingEntryInfoTargetRegion:
4133 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4134 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4135 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4136 /*Order=*/GetMDInt(5));
4137 break;
4138 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4139 OffloadingEntryInfoDeviceGlobalVar:
4140 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4141 /*MangledName=*/GetMDString(1),
4142 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4143 /*Flags=*/GetMDInt(2)),
4144 /*Order=*/GetMDInt(3));
4145 break;
4146 }
4147 }
4148}
4149
4150void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4151 if (!KmpRoutineEntryPtrTy) {
4152 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4153 ASTContext &C = CGM.getContext();
4154 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4155 FunctionProtoType::ExtProtoInfo EPI;
4156 KmpRoutineEntryPtrQTy = C.getPointerType(
4157 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4158 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4159 }
4160}
4161
4162QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4163 // Make sure the type of the entry is already created. This is the type we
4164 // have to create:
4165 // struct __tgt_offload_entry{
4166 // void *addr; // Pointer to the offload entry info.
4167 // // (function or global)
4168 // char *name; // Name of the function or global.
4169 // size_t size; // Size of the entry info (0 if it a function).
4170 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4171 // int32_t reserved; // Reserved, to use by the runtime library.
4172 // };
4173 if (TgtOffloadEntryQTy.isNull()) {
4174 ASTContext &C = CGM.getContext();
4175 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4176 RD->startDefinition();
4177 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4178 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4179 addFieldToRecordDecl(C, RD, C.getSizeType());
4180 addFieldToRecordDecl(
4181 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4182 addFieldToRecordDecl(
4183 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4184 RD->completeDefinition();
4185 RD->addAttr(PackedAttr::CreateImplicit(C));
4186 TgtOffloadEntryQTy = C.getRecordType(RD);
4187 }
4188 return TgtOffloadEntryQTy;
4189}
4190
4191QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4192 // These are the types we need to build:
4193 // struct __tgt_device_image{
4194 // void *ImageStart; // Pointer to the target code start.
4195 // void *ImageEnd; // Pointer to the target code end.
4196 // // We also add the host entries to the device image, as it may be useful
4197 // // for the target runtime to have access to that information.
4198 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
4199 // // the entries.
4200 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4201 // // entries (non inclusive).
4202 // };
4203 if (TgtDeviceImageQTy.isNull()) {
4204 ASTContext &C = CGM.getContext();
4205 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4206 RD->startDefinition();
4207 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4208 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4209 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4210 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4211 RD->completeDefinition();
4212 TgtDeviceImageQTy = C.getRecordType(RD);
4213 }
4214 return TgtDeviceImageQTy;
4215}
4216
4217QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4218 // struct __tgt_bin_desc{
4219 // int32_t NumDevices; // Number of devices supported.
4220 // __tgt_device_image *DeviceImages; // Arrays of device images
4221 // // (one per device).
4222 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
4223 // // entries.
4224 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
4225 // // entries (non inclusive).
4226 // };
4227 if (TgtBinaryDescriptorQTy.isNull()) {
4228 ASTContext &C = CGM.getContext();
4229 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4230 RD->startDefinition();
4231 addFieldToRecordDecl(
4232 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4233 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4234 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4235 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4236 RD->completeDefinition();
4237 TgtBinaryDescriptorQTy = C.getRecordType(RD);
4238 }
4239 return TgtBinaryDescriptorQTy;
4240}
4241
4242namespace {
4243struct PrivateHelpersTy {
4244 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4245 const VarDecl *PrivateElemInit)
4246 : Original(Original), PrivateCopy(PrivateCopy),
4247 PrivateElemInit(PrivateElemInit) {}
4248 const VarDecl *Original;
4249 const VarDecl *PrivateCopy;
4250 const VarDecl *PrivateElemInit;
4251};
4252typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4253} // anonymous namespace
4254
4255static RecordDecl *
4256createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4257 if (!Privates.empty()) {
4258 ASTContext &C = CGM.getContext();
4259 // Build struct .kmp_privates_t. {
4260 // /* private vars */
4261 // };
4262 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4263 RD->startDefinition();
4264 for (const auto &Pair : Privates) {
4265 const VarDecl *VD = Pair.second.Original;
4266 QualType Type = VD->getType().getNonReferenceType();
4267 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4268 if (VD->hasAttrs()) {
4269 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4270 E(VD->getAttrs().end());
4271 I != E; ++I)
4272 FD->addAttr(*I);
4273 }
4274 }
4275 RD->completeDefinition();
4276 return RD;
4277 }
4278 return nullptr;
4279}
4280
4281static RecordDecl *
4282createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4283 QualType KmpInt32Ty,
4284 QualType KmpRoutineEntryPointerQTy) {
4285 ASTContext &C = CGM.getContext();
4286 // Build struct kmp_task_t {
4287 // void * shareds;
4288 // kmp_routine_entry_t routine;
4289 // kmp_int32 part_id;
4290 // kmp_cmplrdata_t data1;
4291 // kmp_cmplrdata_t data2;
4292 // For taskloops additional fields:
4293 // kmp_uint64 lb;
4294 // kmp_uint64 ub;
4295 // kmp_int64 st;
4296 // kmp_int32 liter;
4297 // void * reductions;
4298 // };
4299 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4300 UD->startDefinition();
4301 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4302 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4303 UD->completeDefinition();
4304 QualType KmpCmplrdataTy = C.getRecordType(UD);
4305 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4306 RD->startDefinition();
4307 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4308 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4309 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4310 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4311 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4312 if (isOpenMPTaskLoopDirective(Kind)) {
4313 QualType KmpUInt64Ty =
4314 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4315 QualType KmpInt64Ty =
4316 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4317 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4318 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4319 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4320 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4321 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4322 }
4323 RD->completeDefinition();
4324 return RD;
4325}
4326
4327static RecordDecl *
4328createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4329 ArrayRef<PrivateDataTy> Privates) {
4330 ASTContext &C = CGM.getContext();
4331 // Build struct kmp_task_t_with_privates {
4332 // kmp_task_t task_data;
4333 // .kmp_privates_t. privates;
4334 // };
4335 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4336 RD->startDefinition();
4337 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4338 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4339 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4340 RD->completeDefinition();
4341 return RD;
4342}
4343
4344/// Emit a proxy function which accepts kmp_task_t as the second
4345/// argument.
4346/// \code
4347/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4348/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4349/// For taskloops:
4350/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4351/// tt->reductions, tt->shareds);
4352/// return 0;
4353/// }
4354/// \endcode
4355static llvm::Value *
4356emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4357 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4358 QualType KmpTaskTWithPrivatesPtrQTy,
4359 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4360 QualType SharedsPtrTy, llvm::Value *TaskFunction,
4361 llvm::Value *TaskPrivatesMap) {
4362 ASTContext &C = CGM.getContext();
4363 FunctionArgList Args;
4364 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4365 ImplicitParamDecl::Other);
4366 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4367 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4368 ImplicitParamDecl::Other);
4369 Args.push_back(&GtidArg);
4370 Args.push_back(&TaskTypeArg);
4371 const auto &TaskEntryFnInfo =
4372 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4373 llvm::FunctionType *TaskEntryTy =
4374 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4375 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4376 auto *TaskEntry = llvm::Function::Create(
4377 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4378 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4379 TaskEntry->setDoesNotRecurse();
4380 CodeGenFunction CGF(CGM);
4381 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4382 Loc, Loc);
4383
4384 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4385 // tt,
4386 // For taskloops:
4387 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4388 // tt->task_data.shareds);
4389 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4390 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4391 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4392 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4393 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4394 const auto *KmpTaskTWithPrivatesQTyRD =
4395 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4396 LValue Base =
4397 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4398 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4399 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4400 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4401 llvm::Value *PartidParam = PartIdLVal.getPointer();
4402
4403 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4404 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4405 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4406 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4407 CGF.ConvertTypeForMem(SharedsPtrTy));
4408
4409 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4410 llvm::Value *PrivatesParam;
4411 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4412 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4413 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4414 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4415 } else {
4416 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4417 }
4418
4419 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4420 TaskPrivatesMap,
4421 CGF.Builder
4422 .CreatePointerBitCastOrAddrSpaceCast(
4423 TDBase.getAddress(), CGF.VoidPtrTy)
4424 .getPointer()};
4425 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4426 std::end(CommonArgs));
4427 if (isOpenMPTaskLoopDirective(Kind)) {
4428 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4429 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4430 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4431 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4432 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4433 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4434 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4435 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4436 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4437 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4438 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4439 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4440 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4441 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4442 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4443 CallArgs.push_back(LBParam);
4444 CallArgs.push_back(UBParam);
4445 CallArgs.push_back(StParam);
4446 CallArgs.push_back(LIParam);
4447 CallArgs.push_back(RParam);
4448 }
4449 CallArgs.push_back(SharedsParam);
4450
4451 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4452 CallArgs);
4453 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4454 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4455 CGF.FinishFunction();
4456 return TaskEntry;
4457}
4458
4459static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4460 SourceLocation Loc,
4461 QualType KmpInt32Ty,
4462 QualType KmpTaskTWithPrivatesPtrQTy,
4463 QualType KmpTaskTWithPrivatesQTy) {
4464 ASTContext &C = CGM.getContext();
4465 FunctionArgList Args;
4466 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4467 ImplicitParamDecl::Other);
4468 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4469 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4470 ImplicitParamDecl::Other);
4471 Args.push_back(&GtidArg);
4472 Args.push_back(&TaskTypeArg);
4473 const auto &DestructorFnInfo =
4474 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4475 llvm::FunctionType *DestructorFnTy =
4476 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4477 std::string Name =
4478 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4479 auto *DestructorFn =
4480 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4481 Name, &CGM.getModule());
4482 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4483 DestructorFnInfo);
4484 DestructorFn->setDoesNotRecurse();
4485 CodeGenFunction CGF(CGM);
4486 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4487 Args, Loc, Loc);
4488
4489 LValue Base = CGF.EmitLoadOfPointerLValue(
4490 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4491 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4492 const auto *KmpTaskTWithPrivatesQTyRD =
4493 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4494 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4495 Base = CGF.EmitLValueForField(Base, *FI);
4496 for (const auto *Field :
4497 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4498 if (QualType::DestructionKind DtorKind =
4499 Field->getType().isDestructedType()) {
4500 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4501 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4502 }
4503 }
4504 CGF.FinishFunction();
4505 return DestructorFn;
4506}
4507
4508/// Emit a privates mapping function for correct handling of private and
4509/// firstprivate variables.
4510/// \code
4511/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4512/// **noalias priv1,..., <tyn> **noalias privn) {
4513/// *priv1 = &.privates.priv1;
4514/// ...;
4515/// *privn = &.privates.privn;
4516/// }
4517/// \endcode
4518static llvm::Value *
4519emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4520 ArrayRef<const Expr *> PrivateVars,
4521 ArrayRef<const Expr *> FirstprivateVars,
4522 ArrayRef<const Expr *> LastprivateVars,
4523 QualType PrivatesQTy,
4524 ArrayRef<PrivateDataTy> Privates) {
4525 ASTContext &C = CGM.getContext();
4526 FunctionArgList Args;
4527 ImplicitParamDecl TaskPrivatesArg(
4528 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4529 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4530 ImplicitParamDecl::Other);
4531 Args.push_back(&TaskPrivatesArg);
4532 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4533 unsigned Counter = 1;
4534 for (const Expr *E : PrivateVars) {
4535 Args.push_back(ImplicitParamDecl::Create(
4536 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4537 C.getPointerType(C.getPointerType(E->getType()))
4538 .withConst()
4539 .withRestrict(),
4540 ImplicitParamDecl::Other));
4541 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4542 PrivateVarsPos[VD] = Counter;
4543 ++Counter;
4544 }
4545 for (const Expr *E : FirstprivateVars) {
4546 Args.push_back(ImplicitParamDecl::Create(
4547 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4548 C.getPointerType(C.getPointerType(E->getType()))
4549 .withConst()
4550 .withRestrict(),
4551 ImplicitParamDecl::Other));
4552 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4553 PrivateVarsPos[VD] = Counter;
4554 ++Counter;
4555 }
4556 for (const Expr *E : LastprivateVars) {
4557 Args.push_back(ImplicitParamDecl::Create(
4558 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4559 C.getPointerType(C.getPointerType(E->getType()))
4560 .withConst()
4561 .withRestrict(),
4562 ImplicitParamDecl::Other));
4563 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4564 PrivateVarsPos[VD] = Counter;
4565 ++Counter;
4566 }
4567 const auto &TaskPrivatesMapFnInfo =
4568 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4569 llvm::FunctionType *TaskPrivatesMapTy =
4570 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4571 std::string Name =
4572 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4573 auto *TaskPrivatesMap = llvm::Function::Create(
4574 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4575 &CGM.getModule());
4576 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4577 TaskPrivatesMapFnInfo);
4578 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4579 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4580 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4581 CodeGenFunction CGF(CGM);
4582 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4583 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4584
4585 // *privi = &.privates.privi;
4586 LValue Base = CGF.EmitLoadOfPointerLValue(
4587 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4588 TaskPrivatesArg.getType()->castAs<PointerType>());
4589 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4590 Counter = 0;
4591 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4592 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4593 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4594 LValue RefLVal =
4595 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4596 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4597 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4598 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4599 ++Counter;
4600 }
4601 CGF.FinishFunction();
4602 return TaskPrivatesMap;
4603}
4604
4605static bool stable_sort_comparator(const PrivateDataTy P1,
4606 const PrivateDataTy P2) {
4607 return P1.first > P2.first;
4608}
4609
4610/// Emit initialization for private variables in task-based directives.
4611static void emitPrivatesInit(CodeGenFunction &CGF,
4612 const OMPExecutableDirective &D,
4613 Address KmpTaskSharedsPtr, LValue TDBase,
4614 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4615 QualType SharedsTy, QualType SharedsPtrTy,
4616 const OMPTaskDataTy &Data,
4617 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4618 ASTContext &C = CGF.getContext();
4619 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4620 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4621 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4622 ? OMPD_taskloop
4623 : OMPD_task;
4624 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4625 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4626 LValue SrcBase;
4627 bool IsTargetTask =
4628 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4629 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4630 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4631 // PointersArray and SizesArray. The original variables for these arrays are
4632 // not captured and we get their addresses explicitly.
4633 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4634 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4635 SrcBase = CGF.MakeAddrLValue(
4636 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4637 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4638 SharedsTy);
4639 }
4640 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4641 for (const PrivateDataTy &Pair : Privates) {
4642 const VarDecl *VD = Pair.second.PrivateCopy;
4643 const Expr *Init = VD->getAnyInitializer();
4644 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4645 !CGF.isTrivialInitializer(Init)))) {
4646 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4647 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4648 const VarDecl *OriginalVD = Pair.second.Original;
4649 // Check if the variable is the target-based BasePointersArray,
4650 // PointersArray or SizesArray.
4651 LValue SharedRefLValue;
4652 QualType Type = OriginalVD->getType();
4653 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4654 if (IsTargetTask && !SharedField) {
4655 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4656 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4657 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4658 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4659 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4660 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4661 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4662 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
;
4663 SharedRefLValue =
4664 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4665 } else {
4666 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4667 SharedRefLValue = CGF.MakeAddrLValue(
4668 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4669 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4670 SharedRefLValue.getTBAAInfo());
4671 }
4672 if (Type->isArrayType()) {
4673 // Initialize firstprivate array.
4674 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4675 // Perform simple memcpy.
4676 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4677 } else {
4678 // Initialize firstprivate array using element-by-element
4679 // initialization.
4680 CGF.EmitOMPAggregateAssign(
4681 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4682 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4683 Address SrcElement) {
4684 // Clean up any temporaries needed by the initialization.
4685 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4686 InitScope.addPrivate(
4687 Elem, [SrcElement]() -> Address { return SrcElement; });
4688 (void)InitScope.Privatize();
4689 // Emit initialization for single element.
4690 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4691 CGF, &CapturesInfo);
4692 CGF.EmitAnyExprToMem(Init, DestElement,
4693 Init->getType().getQualifiers(),
4694 /*IsInitializer=*/false);
4695 });
4696 }
4697 } else {
4698 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4699 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4700 return SharedRefLValue.getAddress();
4701 });
4702 (void)InitScope.Privatize();
4703 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4704 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4705 /*capturedByInit=*/false);
4706 }
4707 } else {
4708 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4709 }
4710 }
4711 ++FI;
4712 }
4713}
4714
4715/// Check if duplication function is required for taskloops.
4716static bool checkInitIsRequired(CodeGenFunction &CGF,
4717 ArrayRef<PrivateDataTy> Privates) {
4718 bool InitRequired = false;
4719 for (const PrivateDataTy &Pair : Privates) {
4720 const VarDecl *VD = Pair.second.PrivateCopy;
4721 const Expr *Init = VD->getAnyInitializer();
4722 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4723 !CGF.isTrivialInitializer(Init));
4724 if (InitRequired)
4725 break;
4726 }
4727 return InitRequired;
4728}
4729
4730
4731/// Emit task_dup function (for initialization of
4732/// private/firstprivate/lastprivate vars and last_iter flag)
4733/// \code
4734/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4735/// lastpriv) {
4736/// // setup lastprivate flag
4737/// task_dst->last = lastpriv;
4738/// // could be constructor calls here...
4739/// }
4740/// \endcode
4741static llvm::Value *
4742emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4743 const OMPExecutableDirective &D,
4744 QualType KmpTaskTWithPrivatesPtrQTy,
4745 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4746 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4747 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4748 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4749 ASTContext &C = CGM.getContext();
4750 FunctionArgList Args;
4751 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4752 KmpTaskTWithPrivatesPtrQTy,
4753 ImplicitParamDecl::Other);
4754 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4755 KmpTaskTWithPrivatesPtrQTy,
4756 ImplicitParamDecl::Other);
4757 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4758 ImplicitParamDecl::Other);
4759 Args.push_back(&DstArg);
4760 Args.push_back(&SrcArg);
4761 Args.push_back(&LastprivArg);
4762 const auto &TaskDupFnInfo =
4763 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4764 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4765 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4766 auto *TaskDup = llvm::Function::Create(
4767 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4768 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4769 TaskDup->setDoesNotRecurse();
4770 CodeGenFunction CGF(CGM);
4771 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4772 Loc);
4773
4774 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4775 CGF.GetAddrOfLocalVar(&DstArg),
4776 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4777 // task_dst->liter = lastpriv;
4778 if (WithLastIter) {
4779 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4780 LValue Base = CGF.EmitLValueForField(
4781 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4782 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4783 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4784 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4785 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4786 }
4787
4788 // Emit initial values for private copies (if any).
4789 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4789, __PRETTY_FUNCTION__))
;
4790 Address KmpTaskSharedsPtr = Address::invalid();
4791 if (!Data.FirstprivateVars.empty()) {
4792 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4793 CGF.GetAddrOfLocalVar(&SrcArg),
4794 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4795 LValue Base = CGF.EmitLValueForField(
4796 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4797 KmpTaskSharedsPtr = Address(
4798 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4799 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4800 KmpTaskTShareds)),
4801 Loc),
4802 CGF.getNaturalTypeAlignment(SharedsTy));
4803 }
4804 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4805 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4806 CGF.FinishFunction();
4807 return TaskDup;
4808}
4809
4810/// Checks if destructor function is required to be generated.
4811/// \return true if cleanups are required, false otherwise.
4812static bool
4813checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4814 bool NeedsCleanup = false;
4815 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4816 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4817 for (const FieldDecl *FD : PrivateRD->fields()) {
4818 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4819 if (NeedsCleanup)
4820 break;
4821 }
4822 return NeedsCleanup;
4823}
4824
4825CGOpenMPRuntime::TaskResultTy
4826CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4827 const OMPExecutableDirective &D,
4828 llvm::Value *TaskFunction, QualType SharedsTy,
4829 Address Shareds, const OMPTaskDataTy &Data) {
4830 ASTContext &C = CGM.getContext();
4831 llvm::SmallVector<PrivateDataTy, 4> Privates;
4832 // Aggregate privates and sort them by the alignment.
4833 auto I = Data.PrivateCopies.begin();
4834 for (const Expr *E : Data.PrivateVars) {
4835 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4836 Privates.emplace_back(
4837 C.getDeclAlign(VD),
4838 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4839 /*PrivateElemInit=*/nullptr));
4840 ++I;
4841 }
4842 I = Data.FirstprivateCopies.begin();
4843 auto IElemInitRef = Data.FirstprivateInits.begin();
4844 for (const Expr *E : Data.FirstprivateVars) {
4845 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4846 Privates.emplace_back(
4847 C.getDeclAlign(VD),
4848 PrivateHelpersTy(
4849 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4850 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4851 ++I;
4852 ++IElemInitRef;
4853 }
4854 I = Data.LastprivateCopies.begin();
4855 for (const Expr *E : Data.LastprivateVars) {
4856 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4857 Privates.emplace_back(
4858 C.getDeclAlign(VD),
4859 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4860 /*PrivateElemInit=*/nullptr));
4861 ++I;
4862 }
4863 std::stable_sort(Privates.begin(), Privates.end(), stable_sort_comparator);
4864 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4865 // Build type kmp_routine_entry_t (if not built yet).
4866 emitKmpRoutineEntryT(KmpInt32Ty);
4867 // Build type kmp_task_t (if not built yet).
4868 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4869 if (SavedKmpTaskloopTQTy.isNull()) {
4870 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4871 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4872 }
4873 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4874 } else {
4875 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4878, __PRETTY_FUNCTION__))
4876 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4878, __PRETTY_FUNCTION__))
4877 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4878, __PRETTY_FUNCTION__))
4878 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4878, __PRETTY_FUNCTION__))
;
4879 if (SavedKmpTaskTQTy.isNull()) {
4880 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4881 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4882 }
4883 KmpTaskTQTy = SavedKmpTaskTQTy;
4884 }
4885 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4886 // Build particular struct kmp_task_t for the given task.
4887 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4888 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4889 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4890 QualType KmpTaskTWithPrivatesPtrQTy =
4891 C.getPointerType(KmpTaskTWithPrivatesQTy);
4892 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4893 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4894 KmpTaskTWithPrivatesTy->getPointerTo();
4895 llvm::Value *KmpTaskTWithPrivatesTySize =
4896 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4897 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4898
4899 // Emit initial values for private copies (if any).
4900 llvm::Value *TaskPrivatesMap = nullptr;
4901 llvm::Type *TaskPrivatesMapTy =
4902 std::next(cast<llvm::Function>(TaskFunction)->arg_begin(), 3)->getType();
4903 if (!Privates.empty()) {
4904 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4905 TaskPrivatesMap = emitTaskPrivateMappingFunction(
4906 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
4907 FI->getType(), Privates);
4908 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4909 TaskPrivatesMap, TaskPrivatesMapTy);
4910 } else {
4911 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4912 cast<llvm::PointerType>(TaskPrivatesMapTy));
4913 }
4914 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4915 // kmp_task_t *tt);
4916 llvm::Value *TaskEntry = emitProxyTaskFunction(
4917 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4918 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4919 TaskPrivatesMap);
4920
4921 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4922 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4923 // kmp_routine_entry_t *task_entry);
4924 // Task flags. Format is taken from
4925 // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h,
4926 // description of kmp_tasking_flags struct.
4927 enum {
4928 TiedFlag = 0x1,
4929 FinalFlag = 0x2,
4930 DestructorsFlag = 0x8,
4931 PriorityFlag = 0x20
4932 };
4933 unsigned Flags = Data.Tied ? TiedFlag : 0;
4934 bool NeedsCleanup = false;
4935 if (!Privates.empty()) {
4936 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
4937 if (NeedsCleanup)
4938 Flags = Flags | DestructorsFlag;
4939 }
4940 if (Data.Priority.getInt())
4941 Flags = Flags | PriorityFlag;
4942 llvm::Value *TaskFlags =
4943 Data.Final.getPointer()
4944 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4945 CGF.Builder.getInt32(FinalFlag),
4946 CGF.Builder.getInt32(/*C=*/0))
4947 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4948 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4949 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4950 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
4951 getThreadID(CGF, Loc), TaskFlags,
4952 KmpTaskTWithPrivatesTySize, SharedsSize,
4953 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4954 TaskEntry, KmpRoutineEntryPtrTy)};
4955 llvm::Value *NewTask = CGF.EmitRuntimeCall(
4956 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
4957 llvm::Value *NewTaskNewTaskTTy =
4958 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4959 NewTask, KmpTaskTWithPrivatesPtrTy);
4960 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4961 KmpTaskTWithPrivatesQTy);
4962 LValue TDBase =
4963 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4964 // Fill the data in the resulting kmp_task_t record.
4965 // Copy shareds if there are any.
4966 Address KmpTaskSharedsPtr = Address::invalid();
4967 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4968 KmpTaskSharedsPtr =
4969 Address(CGF.EmitLoadOfScalar(
4970 CGF.EmitLValueForField(
4971 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4972 KmpTaskTShareds)),
4973 Loc),
4974 CGF.getNaturalTypeAlignment(SharedsTy));
4975 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4976 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4977 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4978 }
4979 // Emit initial values for private copies (if any).
4980 TaskResultTy Result;
4981 if (!Privates.empty()) {
4982 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4983 SharedsTy, SharedsPtrTy, Data, Privates,
4984 /*ForDup=*/false);
4985 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4986 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4987 Result.TaskDupFn = emitTaskDupFunction(
4988 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4989 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4990 /*WithLastIter=*/!Data.LastprivateVars.empty());
4991 }
4992 }
4993 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4994 enum { Priority = 0, Destructors = 1 };
4995 // Provide pointer to function with destructors for privates.
4996 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4997 const RecordDecl *KmpCmplrdataUD =
4998 (*FI)->getType()->getAsUnionType()->getDecl();
4999 if (NeedsCleanup) {
5000 llvm::Value *DestructorFn = emitDestructorsFunction(
5001 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5002 KmpTaskTWithPrivatesQTy);
5003 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5004 LValue DestructorsLV = CGF.EmitLValueForField(
5005 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5006 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5007 DestructorFn, KmpRoutineEntryPtrTy),
5008 DestructorsLV);
5009 }
5010 // Set priority.
5011 if (Data.Priority.getInt()) {
5012 LValue Data2LV = CGF.EmitLValueForField(
5013 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5014 LValue PriorityLV = CGF.EmitLValueForField(
5015 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5016 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5017 }
5018 Result.NewTask = NewTask;
5019 Result.TaskEntry = TaskEntry;
5020 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5021 Result.TDBase = TDBase;
5022 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5023 return Result;
5024}
5025
5026void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5027 const OMPExecutableDirective &D,
5028 llvm::Value *TaskFunction,
5029 QualType SharedsTy, Address Shareds,
5030 const Expr *IfCond,
5031 const OMPTaskDataTy &Data) {
5032 if (!CGF.HaveInsertPoint())
5033 return;
5034
5035 TaskResultTy Result =
5036 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5037 llvm::Value *NewTask = Result.NewTask;
5038 llvm::Value *TaskEntry = Result.TaskEntry;
5039 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5040 LValue TDBase = Result.TDBase;
5041 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5042 ASTContext &C = CGM.getContext();
5043 // Process list of dependences.
5044 Address DependenciesArray = Address::invalid();
5045 unsigned NumDependencies = Data.Dependences.size();
5046 if (NumDependencies) {
5047 // Dependence kind for RTL.
5048 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
5049 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5050 RecordDecl *KmpDependInfoRD;
5051 QualType FlagsTy =
5052 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5053 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5054 if (KmpDependInfoTy.isNull()) {
5055 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5056 KmpDependInfoRD->startDefinition();
5057 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5058 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5059 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5060 KmpDependInfoRD->completeDefinition();
5061 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5062 } else {
5063 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5064 }
5065 CharUnits DependencySize = C.getTypeSizeInChars(KmpDependInfoTy);
5066 // Define type kmp_depend_info[<Dependences.size()>];
5067 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5068 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5069 ArrayType::Normal, /*IndexTypeQuals=*/0);
5070 // kmp_depend_info[<Dependences.size()>] deps;
5071 DependenciesArray =
5072 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5073 for (unsigned I = 0; I < NumDependencies; ++I) {
5074 const Expr *E = Data.Dependences[I].second;
5075 LValue Addr = CGF.EmitLValue(E);
5076 llvm::Value *Size;
5077 QualType Ty = E->getType();
5078 if (const auto *ASE =
5079 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5080 LValue UpAddrLVal =
5081 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
5082 llvm::Value *UpAddr =
5083 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5084 llvm::Value *LowIntPtr =
5085 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5086 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5087 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5088 } else {
5089 Size = CGF.getTypeSize(Ty);
5090 }
5091 LValue Base = CGF.MakeAddrLValue(
5092 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I, DependencySize),
5093 KmpDependInfoTy);
5094 // deps[i].base_addr = &<Dependences[i].second>;
5095 LValue BaseAddrLVal = CGF.EmitLValueForField(
5096 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5097 CGF.EmitStoreOfScalar(
5098 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5099 BaseAddrLVal);
5100 // deps[i].len = sizeof(<Dependences[i].second>);
5101 LValue LenLVal = CGF.EmitLValueForField(
5102 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5103 CGF.EmitStoreOfScalar(Size, LenLVal);
5104 // deps[i].flags = <Dependences[i].first>;
5105 RTLDependenceKindTy DepKind;
5106 switch (Data.Dependences[I].first) {
5107 case OMPC_DEPEND_in:
5108 DepKind = DepIn;
5109 break;
5110 // Out and InOut dependencies must use the same code.
5111 case OMPC_DEPEND_out:
5112 case OMPC_DEPEND_inout:
5113 DepKind = DepInOut;
5114 break;
5115 case OMPC_DEPEND_source:
5116 case OMPC_DEPEND_sink:
5117 case OMPC_DEPEND_unknown:
5118 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-8~svn350071/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5118)
;
5119 }
5120 LValue FlagsLVal = CGF.EmitLValueForField(
5121 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5122 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5123 FlagsLVal);
5124 }
5125 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5126 CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()),
5127 CGF.VoidPtrTy);
5128 }
5129
5130 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5131 // libcall.
5132 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5133 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5134 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5135 // list is not empty
5136 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5137 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5138 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5139 llvm::Value *DepTaskArgs[7];
5140 if (NumDependencies) {
5141 DepTaskArgs[0] = UpLoc;
5142 DepTaskArgs[1] = ThreadID;
5143 DepTaskArgs[2] = NewTask;
5144 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5145 DepTaskArgs[4] = DependenciesArray.getPointer();
5146 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5147 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5148 }
5149 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5150 &TaskArgs,
5151 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5152 if (!Data.Tied) {
5153 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5154 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5155 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5156 }
5157 if (NumDependencies) {
5158 CGF.EmitRuntimeCall(
5159 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5160 } else {
5161 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5162 TaskArgs);
5163 }
5164 // Check if parent region is untied and build return for untied task;
5165 if (auto *Region =
5166 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5167 Region->emitUntiedSwitch(CGF);
5168 };
5169
5170 llvm::Value *DepWaitTaskArgs[6];
5171 if (NumDependencies) {
5172 DepWaitTaskArgs[0] = UpLoc;
5173 DepWaitTaskArgs[1] = ThreadID;
5174 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5175 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5176 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5177 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5178 }
5179 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5180 NumDependencies, &DepWaitTaskArgs,
5181 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5182 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5183 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5184 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5185 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5186 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5187 // is specified.
5188 if (NumDependencies)
5189 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5190 DepWaitTaskArgs);
5191 // Call proxy_task_entry(gtid, new_task);
5192 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5193 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5194 Action.Enter(CGF);
5195 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5196 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5197 OutlinedFnArgs);
5198 };
5199
5200 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5201 // kmp_task_t *new_task);
5202 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5203 // kmp_task_t *new_task);
5204 RegionCodeGenTy RCG(CodeGen);
5205 CommonActionTy Action(
5206 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5207 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5208 RCG.setAction(Action);
5209 RCG(CGF);
5210 };
5211
5212 if (IfCond) {
5213 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5214 } else {
5215 RegionCodeGenTy ThenRCG(ThenCodeGen);
5216 ThenRCG(CGF);
5217 }
5218}
5219
5220void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5221 const OMPLoopDirective &D,
5222 llvm::Value *TaskFunction,
5223 QualType SharedsTy, Address Shareds,
5224 const Expr *IfCond,
5225 const OMPTaskDataTy &Data) {
5226 if (!CGF.HaveInsertPoint())
5227 return;
5228 TaskResultTy Result =
5229 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5230 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5231 // libcall.
5232 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5233 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5234 // sched, kmp_uint64 grainsize, void *task_dup);
5235 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5236 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5237 llvm::Value *IfVal;
5238 if (IfCond) {
5239 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5240 /*isSigned=*/true);
5241 } else {
5242 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5243 }
5244
5245 LValue LBLVal = CGF.EmitLValueForField(
5246 Result.TDBase,
5247 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5248 const auto *LBVar =
5249 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5250 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5251 /*IsInitializer=*/true);
5252 LValue UBLVal = CGF.EmitLValueForField(
5253 Result.TDBase,
5254 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5255 const auto *UBVar =
5256 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5257 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5258 /*IsInitializer=*/true);
5259 LValue StLVal = CGF.EmitLValueForField(
5260 Result.TDBase,
5261 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5262 const auto *StVar =
5263 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5264 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5265 /*IsInitializer=*/true);
5266 // Store reductions address.
5267 LValue RedLVal = CGF.EmitLValueForField(
5268 Result.TDBase,
5269 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5270 if (Data.Reductions) {
5271 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5272 } else {
5273 CGF.EmitNullInitialization(RedLVal.getAddress(),
5274 CGF.getContext().VoidPtrTy);
5275 }
5276 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5277 llvm::Value *TaskArgs[] = {
5278 UpLoc,
5279 ThreadID,
5280 Result.NewTask,
5281 IfVal,
5282 LBLVal.getPointer(),
5283 UBLVal.getPointer(),
5284 CGF.EmitLoadOfScalar(StLVal, Loc),
5285 llvm::ConstantInt::getSigned(
5286 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5287 llvm::ConstantInt::getSigned(
5288 CGF.IntTy, Data.Schedule.getPointer()
5289 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5290 : NoSchedule),
5291 Data.Schedule.getPointer()
5292 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5293 /*isSigned=*/false)
5294 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5295 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5296 Result.TaskDupFn, CGF.VoidPtrTy)
5297 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5298 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5299}
5300
5301/// Emit reduction operation for each element of array (required for
5302/// array sections) LHS op = RHS.
5303/// \param Type Type of array.
5304/// \param LHSVar Variable on the left side of the reduction operation
5305/// (references element of array in original variable).
5306/// \param RHSVar Variable on the right side of the reduction operation
5307/// (references element of array in original variable).
5308/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5309/// RHSVar.
5310static void EmitOMPAggregateReduction(
5311 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5312 const VarDecl *RHSVar,
5313 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5314 const Expr *, const Expr *)> &RedOpGen,
5315 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5316 const Expr *UpExpr = nullptr) {
5317 // Perform element-by-element initialization.
5318 QualType ElementTy;
5319 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5320 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5321
5322 // Drill down to the base element type on both arrays.
5323 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5324 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5325
5326 llvm::Value *RHSBegin = RHSAddr.getPointer();
5327 llvm::Value *LHSBegin = LHSAddr.getPointer();
5328 // Cast from pointer to array type to pointer to single element.
5329 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5330 // The basic structure here is a while-do loop.
5331 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5332 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5333 llvm::Value *IsEmpty =
5334 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5335 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5336
5337 // Enter the loop body, making that address the current address.
5338 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5339 CGF.EmitBlock(BodyBB);
5340
5341 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5342
5343 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5344 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5345 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5346 Address RHSElementCurrent =
5347 Address(RHSElementPHI,
5348 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5349
5350 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5351 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5352 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5353 Address LHSElementCurrent =
5354 Address(LHSElementPHI,
5355 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5356
5357 // Emit copy.
5358 CodeGenFunction::OMPPrivateScope Scope(CGF);
5359 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5360 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5361 Scope.Privatize();
5362 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5363 Scope.ForceCleanup();
5364
5365 // Shift the address forward by one element.
5366 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5367 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5368 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5369 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5370 // Check whether we've reached the end.
5371 llvm::Value *Done =
5372 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5373 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5374 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5375 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5376
5377 // Done.
5378 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5379}
5380
5381/// Emit reduction combiner. If the combiner is a simple expression emit it as
5382/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5383/// UDR combiner function.
5384static void emitReductionCombiner(CodeGenFunction &CGF,
5385 const Expr *ReductionOp) {
5386 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5387 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5388 if (const auto *DRE =
5389 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5390 if (const auto *DRD =
5391 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5392 std::pair<llvm::Function *, llvm::Function *> Reduction =
5393 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5394 RValue Func = RValue::get(Reduction.first);
5395 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5396 CGF.EmitIgnoredExpr(ReductionOp);
5397 return;
5398 }
5399 CGF.EmitIgnoredExpr(ReductionOp);
5400}
5401
5402llvm::Value *CGOpenMPRuntime::emitReductionFunction(
5403 CodeGenModule &CGM, SourceLocation Loc, llvm::Type *ArgsType,
5404 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
5405 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
5406 ASTContext &C = CGM.getContext();
5407
5408 // void reduction_func(void *LHSArg, void *RHSArg);
5409 FunctionArgList Args;
5410 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5411 ImplicitParamDecl::Other);
5412 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5413 ImplicitParamDecl::Other);
5414 Args.push_back(&LHSArg);
5415 Args.push_back(&RHSArg);
5416 const auto &CGFI =
5417 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5418 std::string Name = getName({"omp", "reduction", "reduction_func"});
5419 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5420 llvm::GlobalValue::InternalLinkage, Name,
5421 &CGM.getModule());
5422 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5423 Fn->setDoesNotRecurse();
5424 CodeGenFunction CGF(CGM);
5425 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5426
5427 // Dst = (void*[n])(LHSArg);
5428 // Src = (void*[n])(RHSArg);
5429 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5430 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5431 ArgsType), CGF.getPointerAlign());
5432 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5433 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5434 ArgsType), CGF.getPointerAlign());
5435
5436 // ...
5437 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5438 // ...
5439 CodeGenFunction::OMPPrivateScope Scope(CGF);
5440 auto IPriv = Privates.begin();
5441 unsigned Idx = 0;
5442 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5443 const auto *RHSVar =
5444 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5445 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5446 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5447 });
5448 const auto *LHSVar =
5449 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5450 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5451 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5452 });
5453 QualType PrivTy = (*IPriv)->getType();
5454 if (PrivTy->isVariablyModifiedType()) {
5455 // Get array size and emit VLA type.
5456 ++Idx;
5457 Address Elem =
5458 CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize());
5459 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5460 const VariableArrayType *VLA =
5461 CGF.getContext().getAsVariableArrayType(PrivTy);
5462 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5463 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5464 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5465 CGF.EmitVariablyModifiedType(PrivTy);
5466 }
5467 }
5468 Scope.Privatize();
5469 IPriv = Privates.begin();
5470 auto ILHS = LHSExprs.begin();
5471 auto IRHS = RHSExprs.begin();
5472 for (const Expr *E : ReductionOps) {
5473 if ((*IPriv)->getType()->isArrayType()) {
5474 // Emit reduction for array section.
5475 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5476 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5477 EmitOMPAggregateReduction(
5478 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5479 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5480 emitReductionCombiner(CGF, E);
5481 });
5482 } else {
5483 // Emit reduction for array subscript or single variable.
5484 emitReductionCombiner(CGF, E);
5485 }
5486 ++IPriv;
5487 ++ILHS;
5488 ++IRHS;
5489 }
5490 Scope.ForceCleanup();
5491 CGF.FinishFunction();
5492 return Fn;
5493}
5494
5495void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5496 const Expr *ReductionOp,
5497 const Expr *PrivateRef,
5498 const DeclRefExpr *LHS,
5499 const DeclRefExpr *RHS) {
5500 if (PrivateRef->getType()->isArrayType()) {
5501 // Emit reduction for array section.
5502 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5503 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5504 EmitOMPAggregateReduction(
5505 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5506 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5507 emitReductionCombiner(CGF, ReductionOp);
5508 });
5509 } else {
5510 // Emit reduction for array subscript or single variable.
5511 emitReductionCombiner(CGF, ReductionOp);
5512 }
5513}
5514
5515void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5516 ArrayRef<const Expr *> Privates,
5517 ArrayRef<const Expr *> LHSExprs,
5518 ArrayRef<const Expr *> RHSExprs,
5519 ArrayRef<const Expr *> ReductionOps,
5520 ReductionOptionsTy Options) {
5521 if (!CGF.HaveInsertPoint())
5522 return;
5523
5524 bool WithNowait = Options.WithNowait;
5525 bool SimpleReduction = Options.SimpleReduction;
5526
5527 // Next code should be emitted for reduction:
5528 //
5529 // static kmp_critical_name lock = { 0 };
5530 //
5531 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5532 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5533 // ...
5534 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5535 // *(Type<n>-1*)rhs[<n>-1]);
5536 // }
5537 //
5538 // ...
5539 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5540 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5541 // RedList, reduce_func, &<lock>)) {
5542 // case 1:
5543 // ...
5544 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5545 // ...
5546 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5547 // break;
5548 // case 2:
5549 // ...
5550 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5551 // ...
5552 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5553 // break;
5554 // default:;
5555 // }
5556 //
5557 // if SimpleReduction is true, only the next code is generated:
5558 // ...
5559 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5560 // ...
5561
5562 ASTContext &C = CGM.getContext();
5563
5564 if (SimpleReduction) {
5565 CodeGenFunction::RunCleanupsScope Scope(CGF);
5566 auto IPriv = Privates.begin();
5567 auto ILHS = LHSExprs.begin();
5568 auto IRHS = RHSExprs.begin();
5569 for (const Expr *E : ReductionOps) {
5570 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5571 cast<DeclRefExpr>(*IRHS));
5572 ++IPriv;
5573 ++ILHS;
5574 ++IRHS;
5575 }
5576 return;
5577 }
5578
5579 // 1. Build a list of reduction variables.
5580 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5581 auto Size = RHSExprs.size();
5582 for (const Expr *E : Privates) {
5583 if (E->getType()->isVariablyModifiedType())
5584 // Reserve place for array size.
5585 ++Size;
5586 }
5587 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5588 QualType ReductionArrayTy =
5589 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
5590 /*IndexTypeQuals=*/0);
5591 Address ReductionList =
5592 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5593 auto IPriv = Privates.begin();
5594 unsigned Idx = 0;
5595 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5596 Address Elem =
5597 CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize());
5598 CGF.Builder.CreateStore(
5599 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5600 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5601 Elem);
5602 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5603 // Store array size.
5604 ++Idx;
5605 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx,
5606 CGF.getPointerSize());
5607 llvm::Value *Size = CGF.Builder.CreateIntCast(
5608 CGF.getVLASize(
5609 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5610 .NumElts,
5611 CGF.SizeTy, /*isSigned=*/false);
5612 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5613 Elem);
5614 }
5615 }
5616
5617 // 2. Emit reduce_func().
5618 llvm::Value *ReductionFn = emitReductionFunction(
5619 CGM, Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(),
5620 Privates, LHSExprs, RHSExprs, ReductionOps);
5621
5622 // 3. Create static kmp_critical_name lock = { 0 };
5623 std::string Name = getName({"reduction"});
5624 llvm::Value *Lock = getCriticalRegionLock(Name);
5625
5626 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5627 // RedList, reduce_func, &<lock>);
5628 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5629 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5630 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5631 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5632 ReductionList.getPointer(), CGF.VoidPtrTy);
5633 llvm::Value *Args[] = {
5634 IdentTLoc, // ident_t *<loc>
5635 ThreadId, // i32 <gtid>
5636 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5637 ReductionArrayTySize, // size_type sizeof(RedList)
5638 RL, // void *RedList
5639 ReductionFn, // void (*) (void *, void *) <reduce_func>
5640 Lock // kmp_critical_name *&<lock>
5641 };
5642 llvm::Value *Res = CGF.EmitRuntimeCall(
5643 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5644 : OMPRTL__kmpc_reduce),
5645 Args);
5646
5647 // 5. Build switch(res)
5648 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5649 llvm::SwitchInst *SwInst =
5650 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5651
5652 // 6. Build case 1:
5653 // ...
5654 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5655 // ...
5656 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5657 // break;
5658 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5659 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5660 CGF.EmitBlock(Case1BB);
5661
5662 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5663 llvm::Value *EndArgs[] = {
5664 IdentTLoc, // ident_t *<loc>
5665 ThreadId, // i32 <gtid>
5666 Lock // kmp_critical_name *&<lock>
5667 };
5668 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5669 CodeGenFunction &CGF, PrePostActionTy &Action) {
5670 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5671 auto IPriv = Privates.begin();
5672 auto ILHS = LHSExprs.begin();
5673 auto IRHS = RHSExprs.begin();
5674 for (const Expr *E : ReductionOps) {
5675 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5676 cast<DeclRefExpr>(*IRHS));
5677 ++IPriv;
5678 ++ILHS;
5679 ++IRHS;
5680 }
5681 };
5682 RegionCodeGenTy RCG(CodeGen);
5683 CommonActionTy Action(
5684 nullptr, llvm::None,
5685 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5686 : OMPRTL__kmpc_end_reduce),
5687 EndArgs);
5688 RCG.setAction(Action);
5689 RCG(CGF);
5690
5691 CGF.EmitBranch(DefaultBB);
5692
5693 // 7. Build case 2:
5694 // ...
5695 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5696 // ...
5697 // break;
5698 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5699 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5700 CGF.EmitBlock(Case2BB);
5701
5702 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5703 CodeGenFunction &CGF, PrePostActionTy &Action) {
5704 auto ILHS = LHSExprs.begin();
5705 auto IRHS = RHSExprs.begin();
5706 auto IPriv = Privates.begin();
5707 for (const Expr *E : ReductionOps) {
5708 const Expr *XExpr = nullptr;
5709 const Expr *EExpr = nullptr;
5710 const Expr *UpExpr = nullptr;
5711 BinaryOperatorKind BO = BO_Comma;
5712 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5713 if (BO->getOpcode() == BO_Assign) {
5714 XExpr = BO->getLHS();
5715 UpExpr = BO->getRHS();
5716 }
5717 }
5718 // Try to emit update expression as a simple atomic.
5719 const Expr *RHSExpr = UpExpr;
5720 if (RHSExpr) {
5721 // Analyze RHS part of the whole expression.
5722 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5723 RHSExpr->IgnoreParenImpCasts())) {
5724 // If this is a conditional operator, analyze its condition for
5725 // min/max reduction operator.
5726 RHSExpr = ACO->getCond();
5727 }
5728 if (const auto *BORHS =
5729 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5730 EExpr = BORHS->getRHS();
5731 BO = BORHS->getOpcode();
5732 }
5733 }
5734 if (XExpr) {
5735 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5736 auto &&AtomicRedGen = [BO, VD,
5737 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5738 const Expr *EExpr, const Expr *UpExpr) {
5739 LValue X = CGF.EmitLValue(XExpr);
5740 RValue E;
5741 if (EExpr)
5742 E = CGF.EmitAnyExpr(EExpr);
5743 CGF.EmitOMPAtomicSimpleUpdateExpr(
5744 X, E, BO, /*IsXLHSInRHSPart=*/true,
5745 llvm::AtomicOrdering::Monotonic, Loc,
5746 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5747 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5748 PrivateScope.addPrivate(
5749 VD, [&CGF, VD, XRValue, Loc]() {
5750 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5751 CGF.emitOMPSimpleStore(
5752 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5753 VD->getType().getNonReferenceType(), Loc);
5754 return LHSTemp;
5755 });
5756 (void)PrivateScope.Privatize();
5757 return CGF.EmitAnyExpr(UpExpr);
5758 });
5759 };
5760 if ((*IPriv)->getType()->isArrayType()) {
5761 // Emit atomic reduction for array section.
5762 const auto *RHSVar =
5763 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5764 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5765 AtomicRedGen, XExpr, EExpr, UpExpr);
5766 } else {
5767 // Emit atomic reduction for array subscript or single variable.
5768 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5769 }
5770 } else {
5771 // Emit as a critical region.
5772 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5773 const Expr *, const Expr *) {
5774 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5775 std::string Name = RT.getName({"atomic_reduction"});
5776 RT.emitCriticalRegion(
5777 CGF, Name,
5778 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5779 Action.Enter(CGF);
5780 emitReductionCombiner(CGF, E);
5781 },
5782 Loc);
5783 };
5784 if ((*IPriv)->getType()->isArrayType()) {
5785 const auto *LHSVar =
5786 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5787