Bug Summary

File:llvm/include/llvm/IR/IRBuilder.h
Warning:line 2239, column 9
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-03-09-184146-41876-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/OpenMPKinds.h"
25#include "clang/Basic/SourceManager.h"
26#include "clang/CodeGen/ConstantInitBuilder.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/SetOperations.h"
29#include "llvm/ADT/StringExtras.h"
30#include "llvm/Bitcode/BitcodeReader.h"
31#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Value.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40
41using namespace clang;
42using namespace CodeGen;
43using namespace llvm::omp;
44
45namespace {
46/// Base class for handling code generation inside OpenMP regions.
47class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
48public:
49 /// Kinds of OpenMP regions used in codegen.
50 enum CGOpenMPRegionKind {
51 /// Region with outlined function for standalone 'parallel'
52 /// directive.
53 ParallelOutlinedRegion,
54 /// Region with outlined function for standalone 'task' directive.
55 TaskOutlinedRegion,
56 /// Region for constructs that do not require function outlining,
57 /// like 'for', 'sections', 'atomic' etc. directives.
58 InlinedRegion,
59 /// Region with outlined function for standalone 'target' directive.
60 TargetRegion,
61 };
62
63 CGOpenMPRegionInfo(const CapturedStmt &CS,
64 const CGOpenMPRegionKind RegionKind,
65 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
66 bool HasCancel)
67 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
68 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
69
70 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
71 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
72 bool HasCancel)
73 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
74 Kind(Kind), HasCancel(HasCancel) {}
75
76 /// Get a variable or parameter for storing global thread id
77 /// inside OpenMP construct.
78 virtual const VarDecl *getThreadIDVariable() const = 0;
79
80 /// Emit the captured statement body.
81 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
82
83 /// Get an LValue for the current ThreadID variable.
84 /// \return LValue for thread id variable. This LValue always has type int32*.
85 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
86
87 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
88
89 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
90
91 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
92
93 bool hasCancel() const { return HasCancel; }
94
95 static bool classof(const CGCapturedStmtInfo *Info) {
96 return Info->getKind() == CR_OpenMP;
97 }
98
99 ~CGOpenMPRegionInfo() override = default;
100
101protected:
102 CGOpenMPRegionKind RegionKind;
103 RegionCodeGenTy CodeGen;
104 OpenMPDirectiveKind Kind;
105 bool HasCancel;
106};
107
108/// API for captured statement code generation in OpenMP constructs.
109class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
110public:
111 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
112 const RegionCodeGenTy &CodeGen,
113 OpenMPDirectiveKind Kind, bool HasCancel,
114 StringRef HelperName)
115 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
116 HasCancel),
117 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
118 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 118, __PRETTY_FUNCTION__))
;
119 }
120
121 /// Get a variable or parameter for storing global thread id
122 /// inside OpenMP construct.
123 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
124
125 /// Get the name of the capture helper.
126 StringRef getHelperName() const override { return HelperName; }
127
128 static bool classof(const CGCapturedStmtInfo *Info) {
129 return CGOpenMPRegionInfo::classof(Info) &&
130 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
131 ParallelOutlinedRegion;
132 }
133
134private:
135 /// A variable or parameter storing global thread id for OpenMP
136 /// constructs.
137 const VarDecl *ThreadIDVar;
138 StringRef HelperName;
139};
140
141/// API for captured statement code generation in OpenMP constructs.
142class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
143public:
144 class UntiedTaskActionTy final : public PrePostActionTy {
145 bool Untied;
146 const VarDecl *PartIDVar;
147 const RegionCodeGenTy UntiedCodeGen;
148 llvm::SwitchInst *UntiedSwitch = nullptr;
149
150 public:
151 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
152 const RegionCodeGenTy &UntiedCodeGen)
153 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
154 void Enter(CodeGenFunction &CGF) override {
155 if (Untied) {
156 // Emit task switching point.
157 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
158 CGF.GetAddrOfLocalVar(PartIDVar),
159 PartIDVar->getType()->castAs<PointerType>());
160 llvm::Value *Res =
161 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
162 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
163 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
164 CGF.EmitBlock(DoneBB);
165 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
166 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
167 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
168 CGF.Builder.GetInsertBlock());
169 emitUntiedSwitch(CGF);
170 }
171 }
172 void emitUntiedSwitch(CodeGenFunction &CGF) const {
173 if (Untied) {
174 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
175 CGF.GetAddrOfLocalVar(PartIDVar),
176 PartIDVar->getType()->castAs<PointerType>());
177 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
178 PartIdLVal);
179 UntiedCodeGen(CGF);
180 CodeGenFunction::JumpDest CurPoint =
181 CGF.getJumpDestInCurrentScope(".untied.next.");
182 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
183 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
184 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
185 CGF.Builder.GetInsertBlock());
186 CGF.EmitBranchThroughCleanup(CurPoint);
187 CGF.EmitBlock(CurPoint.getBlock());
188 }
189 }
190 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
191 };
192 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
193 const VarDecl *ThreadIDVar,
194 const RegionCodeGenTy &CodeGen,
195 OpenMPDirectiveKind Kind, bool HasCancel,
196 const UntiedTaskActionTy &Action)
197 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
198 ThreadIDVar(ThreadIDVar), Action(Action) {
199 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 199, __PRETTY_FUNCTION__))
;
200 }
201
202 /// Get a variable or parameter for storing global thread id
203 /// inside OpenMP construct.
204 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
205
206 /// Get an LValue for the current ThreadID variable.
207 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
208
209 /// Get the name of the capture helper.
210 StringRef getHelperName() const override { return ".omp_outlined."; }
211
212 void emitUntiedSwitch(CodeGenFunction &CGF) override {
213 Action.emitUntiedSwitch(CGF);
214 }
215
216 static bool classof(const CGCapturedStmtInfo *Info) {
217 return CGOpenMPRegionInfo::classof(Info) &&
218 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
219 TaskOutlinedRegion;
220 }
221
222private:
223 /// A variable or parameter storing global thread id for OpenMP
224 /// constructs.
225 const VarDecl *ThreadIDVar;
226 /// Action for emitting code for untied tasks.
227 const UntiedTaskActionTy &Action;
228};
229
230/// API for inlined captured statement code generation in OpenMP
231/// constructs.
232class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
233public:
234 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
235 const RegionCodeGenTy &CodeGen,
236 OpenMPDirectiveKind Kind, bool HasCancel)
237 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
238 OldCSI(OldCSI),
239 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
240
241 // Retrieve the value of the context parameter.
242 llvm::Value *getContextValue() const override {
243 if (OuterRegionInfo)
244 return OuterRegionInfo->getContextValue();
245 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 245)
;
246 }
247
248 void setContextValue(llvm::Value *V) override {
249 if (OuterRegionInfo) {
250 OuterRegionInfo->setContextValue(V);
251 return;
252 }
253 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 253)
;
254 }
255
256 /// Lookup the captured field decl for a variable.
257 const FieldDecl *lookup(const VarDecl *VD) const override {
258 if (OuterRegionInfo)
259 return OuterRegionInfo->lookup(VD);
260 // If there is no outer outlined region,no need to lookup in a list of
261 // captured variables, we can use the original one.
262 return nullptr;
263 }
264
265 FieldDecl *getThisFieldDecl() const override {
266 if (OuterRegionInfo)
267 return OuterRegionInfo->getThisFieldDecl();
268 return nullptr;
269 }
270
271 /// Get a variable or parameter for storing global thread id
272 /// inside OpenMP construct.
273 const VarDecl *getThreadIDVariable() const override {
274 if (OuterRegionInfo)
275 return OuterRegionInfo->getThreadIDVariable();
276 return nullptr;
277 }
278
279 /// Get an LValue for the current ThreadID variable.
280 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
281 if (OuterRegionInfo)
282 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
283 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 283)
;
284 }
285
286 /// Get the name of the capture helper.
287 StringRef getHelperName() const override {
288 if (auto *OuterRegionInfo = getOldCSI())
289 return OuterRegionInfo->getHelperName();
290 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 290)
;
291 }
292
293 void emitUntiedSwitch(CodeGenFunction &CGF) override {
294 if (OuterRegionInfo)
295 OuterRegionInfo->emitUntiedSwitch(CGF);
296 }
297
298 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
299
300 static bool classof(const CGCapturedStmtInfo *Info) {
301 return CGOpenMPRegionInfo::classof(Info) &&
302 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
303 }
304
305 ~CGOpenMPInlinedRegionInfo() override = default;
306
307private:
308 /// CodeGen info about outer OpenMP region.
309 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
310 CGOpenMPRegionInfo *OuterRegionInfo;
311};
312
313/// API for captured statement code generation in OpenMP target
314/// constructs. For this captures, implicit parameters are used instead of the
315/// captured fields. The name of the target region has to be unique in a given
316/// application so it is provided by the client, because only the client has
317/// the information to generate that.
318class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
319public:
320 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
321 const RegionCodeGenTy &CodeGen, StringRef HelperName)
322 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
323 /*HasCancel=*/false),
324 HelperName(HelperName) {}
325
326 /// This is unused for target regions because each starts executing
327 /// with a single thread.
328 const VarDecl *getThreadIDVariable() const override { return nullptr; }
329
330 /// Get the name of the capture helper.
331 StringRef getHelperName() const override { return HelperName; }
332
333 static bool classof(const CGCapturedStmtInfo *Info) {
334 return CGOpenMPRegionInfo::classof(Info) &&
335 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
336 }
337
338private:
339 StringRef HelperName;
340};
341
342static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
343 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 343)
;
344}
345/// API for generation of expressions captured in a innermost OpenMP
346/// region.
347class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
348public:
349 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
350 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
351 OMPD_unknown,
352 /*HasCancel=*/false),
353 PrivScope(CGF) {
354 // Make sure the globals captured in the provided statement are local by
355 // using the privatization logic. We assume the same variable is not
356 // captured more than once.
357 for (const auto &C : CS.captures()) {
358 if (!C.capturesVariable() && !C.capturesVariableByCopy())
359 continue;
360
361 const VarDecl *VD = C.getCapturedVar();
362 if (VD->isLocalVarDeclOrParm())
363 continue;
364
365 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
366 /*RefersToEnclosingVariableOrCapture=*/false,
367 VD->getType().getNonReferenceType(), VK_LValue,
368 C.getLocation());
369 PrivScope.addPrivate(
370 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
371 }
372 (void)PrivScope.Privatize();
373 }
374
375 /// Lookup the captured field decl for a variable.
376 const FieldDecl *lookup(const VarDecl *VD) const override {
377 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
378 return FD;
379 return nullptr;
380 }
381
382 /// Emit the captured statement body.
383 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
384 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 384)
;
385 }
386
387 /// Get a variable or parameter for storing global thread id
388 /// inside OpenMP construct.
389 const VarDecl *getThreadIDVariable() const override {
390 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 390)
;
391 }
392
393 /// Get the name of the capture helper.
394 StringRef getHelperName() const override {
395 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 395)
;
396 }
397
398 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
399
400private:
401 /// Private scope to capture global variables.
402 CodeGenFunction::OMPPrivateScope PrivScope;
403};
404
405/// RAII for emitting code of OpenMP constructs.
406class InlinedOpenMPRegionRAII {
407 CodeGenFunction &CGF;
408 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
409 FieldDecl *LambdaThisCaptureField = nullptr;
410 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
411
412public:
413 /// Constructs region for combined constructs.
414 /// \param CodeGen Code generation sequence for combined directives. Includes
415 /// a list of functions used for code generation of implicitly inlined
416 /// regions.
417 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
418 OpenMPDirectiveKind Kind, bool HasCancel)
419 : CGF(CGF) {
420 // Start emission for the construct.
421 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
422 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
423 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
424 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
425 CGF.LambdaThisCaptureField = nullptr;
426 BlockInfo = CGF.BlockInfo;
427 CGF.BlockInfo = nullptr;
428 }
429
430 ~InlinedOpenMPRegionRAII() {
431 // Restore original CapturedStmtInfo only if we're done with code emission.
432 auto *OldCSI =
433 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
434 delete CGF.CapturedStmtInfo;
435 CGF.CapturedStmtInfo = OldCSI;
436 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
437 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
438 CGF.BlockInfo = BlockInfo;
439 }
440};
441
442/// Values for bit flags used in the ident_t to describe the fields.
443/// All enumeric elements are named and described in accordance with the code
444/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
445enum OpenMPLocationFlags : unsigned {
446 /// Use trampoline for internal microtask.
447 OMP_IDENT_IMD = 0x01,
448 /// Use c-style ident structure.
449 OMP_IDENT_KMPC = 0x02,
450 /// Atomic reduction option for kmpc_reduce.
451 OMP_ATOMIC_REDUCE = 0x10,
452 /// Explicit 'barrier' directive.
453 OMP_IDENT_BARRIER_EXPL = 0x20,
454 /// Implicit barrier in code.
455 OMP_IDENT_BARRIER_IMPL = 0x40,
456 /// Implicit barrier in 'for' directive.
457 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
458 /// Implicit barrier in 'sections' directive.
459 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
460 /// Implicit barrier in 'single' directive.
461 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
462 /// Call of __kmp_for_static_init for static loop.
463 OMP_IDENT_WORK_LOOP = 0x200,
464 /// Call of __kmp_for_static_init for sections.
465 OMP_IDENT_WORK_SECTIONS = 0x400,
466 /// Call of __kmp_for_static_init for distribute.
467 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
468 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
469};
470
471namespace {
472LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
473/// Values for bit flags for marking which requires clauses have been used.
474enum OpenMPOffloadingRequiresDirFlags : int64_t {
475 /// flag undefined.
476 OMP_REQ_UNDEFINED = 0x000,
477 /// no requires clause present.
478 OMP_REQ_NONE = 0x001,
479 /// reverse_offload clause.
480 OMP_REQ_REVERSE_OFFLOAD = 0x002,
481 /// unified_address clause.
482 OMP_REQ_UNIFIED_ADDRESS = 0x004,
483 /// unified_shared_memory clause.
484 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
485 /// dynamic_allocators clause.
486 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
487 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
488};
489
490enum OpenMPOffloadingReservedDeviceIDs {
491 /// Device ID if the device was not defined, runtime should get it
492 /// from environment variables in the spec.
493 OMP_DEVICEID_UNDEF = -1,
494};
495} // anonymous namespace
496
497/// Describes ident structure that describes a source location.
498/// All descriptions are taken from
499/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
500/// Original structure:
501/// typedef struct ident {
502/// kmp_int32 reserved_1; /**< might be used in Fortran;
503/// see above */
504/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
505/// KMP_IDENT_KMPC identifies this union
506/// member */
507/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
508/// see above */
509///#if USE_ITT_BUILD
510/// /* but currently used for storing
511/// region-specific ITT */
512/// /* contextual information. */
513///#endif /* USE_ITT_BUILD */
514/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
515/// C++ */
516/// char const *psource; /**< String describing the source location.
517/// The string is composed of semi-colon separated
518// fields which describe the source file,
519/// the function and a pair of line numbers that
520/// delimit the construct.
521/// */
522/// } ident_t;
523enum IdentFieldIndex {
524 /// might be used in Fortran
525 IdentField_Reserved_1,
526 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
527 IdentField_Flags,
528 /// Not really used in Fortran any more
529 IdentField_Reserved_2,
530 /// Source[4] in Fortran, do not use for C++
531 IdentField_Reserved_3,
532 /// String describing the source location. The string is composed of
533 /// semi-colon separated fields which describe the source file, the function
534 /// and a pair of line numbers that delimit the construct.
535 IdentField_PSource
536};
537
538/// Schedule types for 'omp for' loops (these enumerators are taken from
539/// the enum sched_type in kmp.h).
540enum OpenMPSchedType {
541 /// Lower bound for default (unordered) versions.
542 OMP_sch_lower = 32,
543 OMP_sch_static_chunked = 33,
544 OMP_sch_static = 34,
545 OMP_sch_dynamic_chunked = 35,
546 OMP_sch_guided_chunked = 36,
547 OMP_sch_runtime = 37,
548 OMP_sch_auto = 38,
549 /// static with chunk adjustment (e.g., simd)
550 OMP_sch_static_balanced_chunked = 45,
551 /// Lower bound for 'ordered' versions.
552 OMP_ord_lower = 64,
553 OMP_ord_static_chunked = 65,
554 OMP_ord_static = 66,
555 OMP_ord_dynamic_chunked = 67,
556 OMP_ord_guided_chunked = 68,
557 OMP_ord_runtime = 69,
558 OMP_ord_auto = 70,
559 OMP_sch_default = OMP_sch_static,
560 /// dist_schedule types
561 OMP_dist_sch_static_chunked = 91,
562 OMP_dist_sch_static = 92,
563 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
564 /// Set if the monotonic schedule modifier was present.
565 OMP_sch_modifier_monotonic = (1 << 29),
566 /// Set if the nonmonotonic schedule modifier was present.
567 OMP_sch_modifier_nonmonotonic = (1 << 30),
568};
569
570enum OpenMPRTLFunction {
571 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
572 /// kmpc_micro microtask, ...);
573 OMPRTL__kmpc_fork_call,
574 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
575 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
576 OMPRTL__kmpc_threadprivate_cached,
577 /// Call to void __kmpc_threadprivate_register( ident_t *,
578 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
579 OMPRTL__kmpc_threadprivate_register,
580 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
581 OMPRTL__kmpc_global_thread_num,
582 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
583 // kmp_critical_name *crit);
584 OMPRTL__kmpc_critical,
585 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
586 // global_tid, kmp_critical_name *crit, uintptr_t hint);
587 OMPRTL__kmpc_critical_with_hint,
588 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
589 // kmp_critical_name *crit);
590 OMPRTL__kmpc_end_critical,
591 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
592 // global_tid);
593 OMPRTL__kmpc_cancel_barrier,
594 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
595 OMPRTL__kmpc_barrier,
596 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
597 OMPRTL__kmpc_for_static_fini,
598 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
599 // global_tid);
600 OMPRTL__kmpc_serialized_parallel,
601 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
602 // global_tid);
603 OMPRTL__kmpc_end_serialized_parallel,
604 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
605 // kmp_int32 num_threads);
606 OMPRTL__kmpc_push_num_threads,
607 // Call to void __kmpc_flush(ident_t *loc);
608 OMPRTL__kmpc_flush,
609 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
610 OMPRTL__kmpc_master,
611 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
612 OMPRTL__kmpc_end_master,
613 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
614 // int end_part);
615 OMPRTL__kmpc_omp_taskyield,
616 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
617 OMPRTL__kmpc_single,
618 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
619 OMPRTL__kmpc_end_single,
620 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
621 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
622 // kmp_routine_entry_t *task_entry);
623 OMPRTL__kmpc_omp_task_alloc,
624 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
625 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
626 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
627 // kmp_int64 device_id);
628 OMPRTL__kmpc_omp_target_task_alloc,
629 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
630 // new_task);
631 OMPRTL__kmpc_omp_task,
632 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
633 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
634 // kmp_int32 didit);
635 OMPRTL__kmpc_copyprivate,
636 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
637 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
638 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
639 OMPRTL__kmpc_reduce,
640 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
641 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
642 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
643 // *lck);
644 OMPRTL__kmpc_reduce_nowait,
645 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
646 // kmp_critical_name *lck);
647 OMPRTL__kmpc_end_reduce,
648 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
649 // kmp_critical_name *lck);
650 OMPRTL__kmpc_end_reduce_nowait,
651 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
652 // kmp_task_t * new_task);
653 OMPRTL__kmpc_omp_task_begin_if0,
654 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
655 // kmp_task_t * new_task);
656 OMPRTL__kmpc_omp_task_complete_if0,
657 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
658 OMPRTL__kmpc_ordered,
659 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
660 OMPRTL__kmpc_end_ordered,
661 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
662 // global_tid);
663 OMPRTL__kmpc_omp_taskwait,
664 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
665 OMPRTL__kmpc_taskgroup,
666 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
667 OMPRTL__kmpc_end_taskgroup,
668 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
669 // int proc_bind);
670 OMPRTL__kmpc_push_proc_bind,
671 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
672 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
673 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
674 OMPRTL__kmpc_omp_task_with_deps,
675 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
676 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
677 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
678 OMPRTL__kmpc_omp_wait_deps,
679 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
680 // global_tid, kmp_int32 cncl_kind);
681 OMPRTL__kmpc_cancellationpoint,
682 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
683 // kmp_int32 cncl_kind);
684 OMPRTL__kmpc_cancel,
685 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
686 // kmp_int32 num_teams, kmp_int32 thread_limit);
687 OMPRTL__kmpc_push_num_teams,
688 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
689 // microtask, ...);
690 OMPRTL__kmpc_fork_teams,
691 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
692 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
693 // sched, kmp_uint64 grainsize, void *task_dup);
694 OMPRTL__kmpc_taskloop,
695 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
696 // num_dims, struct kmp_dim *dims);
697 OMPRTL__kmpc_doacross_init,
698 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
699 OMPRTL__kmpc_doacross_fini,
700 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
701 // *vec);
702 OMPRTL__kmpc_doacross_post,
703 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
704 // *vec);
705 OMPRTL__kmpc_doacross_wait,
706 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
707 // *data);
708 OMPRTL__kmpc_task_reduction_init,
709 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
710 // *d);
711 OMPRTL__kmpc_task_reduction_get_th_data,
712 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
713 OMPRTL__kmpc_alloc,
714 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
715 OMPRTL__kmpc_free,
716
717 //
718 // Offloading related calls
719 //
720 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
721 // size);
722 OMPRTL__kmpc_push_target_tripcount,
723 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
724 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
725 // *arg_types);
726 OMPRTL__tgt_target,
727 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
728 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
729 // *arg_types);
730 OMPRTL__tgt_target_nowait,
731 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
732 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
733 // *arg_types, int32_t num_teams, int32_t thread_limit);
734 OMPRTL__tgt_target_teams,
735 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
736 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
737 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
738 OMPRTL__tgt_target_teams_nowait,
739 // Call to void __tgt_register_requires(int64_t flags);
740 OMPRTL__tgt_register_requires,
741 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
742 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
743 OMPRTL__tgt_target_data_begin,
744 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
745 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
746 // *arg_types);
747 OMPRTL__tgt_target_data_begin_nowait,
748 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
749 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
750 OMPRTL__tgt_target_data_end,
751 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
752 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
753 // *arg_types);
754 OMPRTL__tgt_target_data_end_nowait,
755 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
756 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
757 OMPRTL__tgt_target_data_update,
758 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
759 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
760 // *arg_types);
761 OMPRTL__tgt_target_data_update_nowait,
762 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
763 OMPRTL__tgt_mapper_num_components,
764 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
765 // *base, void *begin, int64_t size, int64_t type);
766 OMPRTL__tgt_push_mapper_component,
767};
768
769/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
770/// region.
771class CleanupTy final : public EHScopeStack::Cleanup {
772 PrePostActionTy *Action;
773
774public:
775 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
776 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
777 if (!CGF.HaveInsertPoint())
778 return;
779 Action->Exit(CGF);
780 }
781};
782
783} // anonymous namespace
784
785void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
786 CodeGenFunction::RunCleanupsScope Scope(CGF);
787 if (PrePostAction) {
788 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
789 Callback(CodeGen, CGF, *PrePostAction);
790 } else {
791 PrePostActionTy Action;
792 Callback(CodeGen, CGF, Action);
793 }
794}
795
796/// Check if the combiner is a call to UDR combiner and if it is so return the
797/// UDR decl used for reduction.
798static const OMPDeclareReductionDecl *
799getReductionInit(const Expr *ReductionOp) {
800 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
801 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
802 if (const auto *DRE =
803 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
804 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
805 return DRD;
806 return nullptr;
807}
808
809static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
810 const OMPDeclareReductionDecl *DRD,
811 const Expr *InitOp,
812 Address Private, Address Original,
813 QualType Ty) {
814 if (DRD->getInitializer()) {
815 std::pair<llvm::Function *, llvm::Function *> Reduction =
816 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
817 const auto *CE = cast<CallExpr>(InitOp);
818 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
819 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
820 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
821 const auto *LHSDRE =
822 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
823 const auto *RHSDRE =
824 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
825 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
826 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
827 [=]() { return Private; });
828 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
829 [=]() { return Original; });
830 (void)PrivateScope.Privatize();
831 RValue Func = RValue::get(Reduction.second);
832 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
833 CGF.EmitIgnoredExpr(InitOp);
834 } else {
835 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
836 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
837 auto *GV = new llvm::GlobalVariable(
838 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
839 llvm::GlobalValue::PrivateLinkage, Init, Name);
840 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
841 RValue InitRVal;
842 switch (CGF.getEvaluationKind(Ty)) {
843 case TEK_Scalar:
844 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
845 break;
846 case TEK_Complex:
847 InitRVal =
848 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
849 break;
850 case TEK_Aggregate:
851 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
852 break;
853 }
854 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
855 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
856 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
857 /*IsInitializer=*/false);
858 }
859}
860
861/// Emit initialization of arrays of complex types.
862/// \param DestAddr Address of the array.
863/// \param Type Type of array.
864/// \param Init Initial expression of array.
865/// \param SrcAddr Address of the original array.
866static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
867 QualType Type, bool EmitDeclareReductionInit,
868 const Expr *Init,
869 const OMPDeclareReductionDecl *DRD,
870 Address SrcAddr = Address::invalid()) {
871 // Perform element-by-element initialization.
872 QualType ElementTy;
873
874 // Drill down to the base element type on both arrays.
875 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
876 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
877 DestAddr =
878 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
879 if (DRD)
880 SrcAddr =
881 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
882
883 llvm::Value *SrcBegin = nullptr;
884 if (DRD)
885 SrcBegin = SrcAddr.getPointer();
886 llvm::Value *DestBegin = DestAddr.getPointer();
887 // Cast from pointer to array type to pointer to single element.
888 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
889 // The basic structure here is a while-do loop.
890 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
891 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
892 llvm::Value *IsEmpty =
893 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
894 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
895
896 // Enter the loop body, making that address the current address.
897 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
898 CGF.EmitBlock(BodyBB);
899
900 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
901
902 llvm::PHINode *SrcElementPHI = nullptr;
903 Address SrcElementCurrent = Address::invalid();
904 if (DRD) {
905 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
906 "omp.arraycpy.srcElementPast");
907 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
908 SrcElementCurrent =
909 Address(SrcElementPHI,
910 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
911 }
912 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
913 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
914 DestElementPHI->addIncoming(DestBegin, EntryBB);
915 Address DestElementCurrent =
916 Address(DestElementPHI,
917 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
918
919 // Emit copy.
920 {
921 CodeGenFunction::RunCleanupsScope InitScope(CGF);
922 if (EmitDeclareReductionInit) {
923 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
924 SrcElementCurrent, ElementTy);
925 } else
926 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
927 /*IsInitializer=*/false);
928 }
929
930 if (DRD) {
931 // Shift the address forward by one element.
932 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
933 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
934 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
935 }
936
937 // Shift the address forward by one element.
938 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
939 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
940 // Check whether we've reached the end.
941 llvm::Value *Done =
942 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
943 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
944 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
945
946 // Done.
947 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
948}
949
950LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
951 return CGF.EmitOMPSharedLValue(E);
952}
953
954LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
955 const Expr *E) {
956 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
957 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
958 return LValue();
959}
960
961void ReductionCodeGen::emitAggregateInitialization(
962 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
963 const OMPDeclareReductionDecl *DRD) {
964 // Emit VarDecl with copy init for arrays.
965 // Get the address of the original variable captured in current
966 // captured region.
967 const auto *PrivateVD =
968 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
969 bool EmitDeclareReductionInit =
970 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
971 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
972 EmitDeclareReductionInit,
973 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
974 : PrivateVD->getInit(),
975 DRD, SharedLVal.getAddress(CGF));
976}
977
978ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
979 ArrayRef<const Expr *> Privates,
980 ArrayRef<const Expr *> ReductionOps) {
981 ClausesData.reserve(Shareds.size());
982 SharedAddresses.reserve(Shareds.size());
983 Sizes.reserve(Shareds.size());
984 BaseDecls.reserve(Shareds.size());
985 auto IPriv = Privates.begin();
986 auto IRed = ReductionOps.begin();
987 for (const Expr *Ref : Shareds) {
988 ClausesData.emplace_back(Ref, *IPriv, *IRed);
989 std::advance(IPriv, 1);
990 std::advance(IRed, 1);
991 }
992}
993
994void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
995 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 996, __PRETTY_FUNCTION__))
996 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 996, __PRETTY_FUNCTION__))
;
997 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
998 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
999 SharedAddresses.emplace_back(First, Second);
1000}
1001
1002void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
1003 const auto *PrivateVD =
1004 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1005 QualType PrivateType = PrivateVD->getType();
1006 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1007 if (!PrivateType->isVariablyModifiedType()) {
1008 Sizes.emplace_back(
1009 CGF.getTypeSize(
1010 SharedAddresses[N].first.getType().getNonReferenceType()),
1011 nullptr);
1012 return;
1013 }
1014 llvm::Value *Size;
1015 llvm::Value *SizeInChars;
1016 auto *ElemType = cast<llvm::PointerType>(
1017 SharedAddresses[N].first.getPointer(CGF)->getType())
1018 ->getElementType();
1019 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1020 if (AsArraySection) {
1021 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1022 SharedAddresses[N].first.getPointer(CGF));
1023 Size = CGF.Builder.CreateNUWAdd(
1024 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1025 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1026 } else {
1027 SizeInChars = CGF.getTypeSize(
1028 SharedAddresses[N].first.getType().getNonReferenceType());
1029 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1030 }
1031 Sizes.emplace_back(SizeInChars, Size);
1032 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1033 CGF,
1034 cast<OpaqueValueExpr>(
1035 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1036 RValue::get(Size));
1037 CGF.EmitVariablyModifiedType(PrivateType);
1038}
1039
1040void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1041 llvm::Value *Size) {
1042 const auto *PrivateVD =
1043 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1044 QualType PrivateType = PrivateVD->getType();
1045 if (!PrivateType->isVariablyModifiedType()) {
1046 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1048, __PRETTY_FUNCTION__))
1047 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1048, __PRETTY_FUNCTION__))
1048 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1048, __PRETTY_FUNCTION__))
;
1049 return;
1050 }
1051 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1052 CGF,
1053 cast<OpaqueValueExpr>(
1054 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1055 RValue::get(Size));
1056 CGF.EmitVariablyModifiedType(PrivateType);
1057}
1058
1059void ReductionCodeGen::emitInitialization(
1060 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1061 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1062 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1062, __PRETTY_FUNCTION__))
;
1063 const auto *PrivateVD =
1064 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1065 const OMPDeclareReductionDecl *DRD =
1066 getReductionInit(ClausesData[N].ReductionOp);
1067 QualType PrivateType = PrivateVD->getType();
1068 PrivateAddr = CGF.Builder.CreateElementBitCast(
1069 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1070 QualType SharedType = SharedAddresses[N].first.getType();
1071 SharedLVal = CGF.MakeAddrLValue(
1072 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
1073 CGF.ConvertTypeForMem(SharedType)),
1074 SharedType, SharedAddresses[N].first.getBaseInfo(),
1075 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1076 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1077 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1078 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1079 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1080 PrivateAddr, SharedLVal.getAddress(CGF),
1081 SharedLVal.getType());
1082 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1083 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1084 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1085 PrivateVD->getType().getQualifiers(),
1086 /*IsInitializer=*/false);
1087 }
1088}
1089
1090bool ReductionCodeGen::needCleanups(unsigned N) {
1091 const auto *PrivateVD =
1092 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1093 QualType PrivateType = PrivateVD->getType();
1094 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1095 return DTorKind != QualType::DK_none;
1096}
1097
1098void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1099 Address PrivateAddr) {
1100 const auto *PrivateVD =
1101 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1102 QualType PrivateType = PrivateVD->getType();
1103 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1104 if (needCleanups(N)) {
1105 PrivateAddr = CGF.Builder.CreateElementBitCast(
1106 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1107 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1108 }
1109}
1110
1111static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1112 LValue BaseLV) {
1113 BaseTy = BaseTy.getNonReferenceType();
1114 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1115 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1116 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1117 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
1118 } else {
1119 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
1120 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1121 }
1122 BaseTy = BaseTy->getPointeeType();
1123 }
1124 return CGF.MakeAddrLValue(
1125 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
1126 CGF.ConvertTypeForMem(ElTy)),
1127 BaseLV.getType(), BaseLV.getBaseInfo(),
1128 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1129}
1130
1131static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1132 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1133 llvm::Value *Addr) {
1134 Address Tmp = Address::invalid();
1135 Address TopTmp = Address::invalid();
1136 Address MostTopTmp = Address::invalid();
1137 BaseTy = BaseTy.getNonReferenceType();
1138 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1139 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1140 Tmp = CGF.CreateMemTemp(BaseTy);
1141 if (TopTmp.isValid())
1142 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1143 else
1144 MostTopTmp = Tmp;
1145 TopTmp = Tmp;
1146 BaseTy = BaseTy->getPointeeType();
1147 }
1148 llvm::Type *Ty = BaseLVType;
1149 if (Tmp.isValid())
1150 Ty = Tmp.getElementType();
1151 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1152 if (Tmp.isValid()) {
1153 CGF.Builder.CreateStore(Addr, Tmp);
1154 return MostTopTmp;
1155 }
1156 return Address(Addr, BaseLVAlignment);
1157}
1158
1159static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1160 const VarDecl *OrigVD = nullptr;
1161 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1162 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1163 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1164 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1165 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1166 Base = TempASE->getBase()->IgnoreParenImpCasts();
1167 DE = cast<DeclRefExpr>(Base);
1168 OrigVD = cast<VarDecl>(DE->getDecl());
1169 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1170 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1171 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1172 Base = TempASE->getBase()->IgnoreParenImpCasts();
1173 DE = cast<DeclRefExpr>(Base);
1174 OrigVD = cast<VarDecl>(DE->getDecl());
1175 }
1176 return OrigVD;
1177}
1178
1179Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1180 Address PrivateAddr) {
1181 const DeclRefExpr *DE;
1182 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1183 BaseDecls.emplace_back(OrigVD);
1184 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1185 LValue BaseLValue =
1186 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1187 OriginalBaseLValue);
1188 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1189 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1190 llvm::Value *PrivatePointer =
1191 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1192 PrivateAddr.getPointer(),
1193 SharedAddresses[N].first.getAddress(CGF).getType());
1194 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1195 return castToBase(CGF, OrigVD->getType(),
1196 SharedAddresses[N].first.getType(),
1197 OriginalBaseLValue.getAddress(CGF).getType(),
1198 OriginalBaseLValue.getAlignment(), Ptr);
1199 }
1200 BaseDecls.emplace_back(
1201 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1202 return PrivateAddr;
1203}
1204
1205bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1206 const OMPDeclareReductionDecl *DRD =
1207 getReductionInit(ClausesData[N].ReductionOp);
1208 return DRD && DRD->getInitializer();
1209}
1210
1211LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1212 return CGF.EmitLoadOfPointerLValue(
1213 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1214 getThreadIDVariable()->getType()->castAs<PointerType>());
1215}
1216
1217void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1218 if (!CGF.HaveInsertPoint())
1219 return;
1220 // 1.2.2 OpenMP Language Terminology
1221 // Structured block - An executable statement with a single entry at the
1222 // top and a single exit at the bottom.
1223 // The point of exit cannot be a branch out of the structured block.
1224 // longjmp() and throw() must not violate the entry/exit criteria.
1225 CGF.EHStack.pushTerminate();
1226 CodeGen(CGF);
1227 CGF.EHStack.popTerminate();
1228}
1229
1230LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1231 CodeGenFunction &CGF) {
1232 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1233 getThreadIDVariable()->getType(),
1234 AlignmentSource::Decl);
1235}
1236
1237static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1238 QualType FieldTy) {
1239 auto *Field = FieldDecl::Create(
1240 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1241 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1242 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1243 Field->setAccess(AS_public);
1244 DC->addDecl(Field);
1245 return Field;
1246}
1247
1248CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1249 StringRef Separator)
1250 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1251 OffloadEntriesInfoManager(CGM) {
1252 ASTContext &C = CGM.getContext();
1253 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1254 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1255 RD->startDefinition();
1256 // reserved_1
1257 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1258 // flags
1259 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1260 // reserved_2
1261 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1262 // reserved_3
1263 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1264 // psource
1265 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1266 RD->completeDefinition();
1267 IdentQTy = C.getRecordType(RD);
1268 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1269 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1270
1271 loadOffloadInfoMetadata();
1272}
1273
1274bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1275 const GlobalDecl &OldGD,
1276 llvm::GlobalValue *OrigAddr,
1277 bool IsForDefinition) {
1278 // Emit at least a definition for the aliasee if the the address of the
1279 // original function is requested.
1280 if (IsForDefinition || OrigAddr)
1281 (void)CGM.GetAddrOfGlobal(NewGD);
1282 StringRef NewMangledName = CGM.getMangledName(NewGD);
1283 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1284 if (Addr && !Addr->isDeclaration()) {
1285 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1286 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(NewGD);
1287 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1288
1289 // Create a reference to the named value. This ensures that it is emitted
1290 // if a deferred decl.
1291 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1292
1293 // Create the new alias itself, but don't set a name yet.
1294 auto *GA =
1295 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1296
1297 if (OrigAddr) {
1298 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1298, __PRETTY_FUNCTION__))
;
1299
1300 GA->takeName(OrigAddr);
1301 OrigAddr->replaceAllUsesWith(
1302 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1303 OrigAddr->eraseFromParent();
1304 } else {
1305 GA->setName(CGM.getMangledName(OldGD));
1306 }
1307
1308 // Set attributes which are particular to an alias; this is a
1309 // specialization of the attributes which may be set on a global function.
1310 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1311 D->isWeakImported())
1312 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1313
1314 CGM.SetCommonAttributes(OldGD, GA);
1315 return true;
1316 }
1317 return false;
1318}
1319
1320void CGOpenMPRuntime::clear() {
1321 InternalVars.clear();
1322 // Clean non-target variable declarations possibly used only in debug info.
1323 for (const auto &Data : EmittedNonTargetVariables) {
1324 if (!Data.getValue().pointsToAliveValue())
1325 continue;
1326 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1327 if (!GV)
1328 continue;
1329 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1330 continue;
1331 GV->eraseFromParent();
1332 }
1333 // Emit aliases for the deferred aliasees.
1334 for (const auto &Pair : DeferredVariantFunction) {
1335 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1336 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1337 // If not able to emit alias, just emit original declaration.
1338 (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1339 /*IsForDefinition=*/false);
1340 }
1341}
1342
1343std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1344 SmallString<128> Buffer;
1345 llvm::raw_svector_ostream OS(Buffer);
1346 StringRef Sep = FirstSeparator;
1347 for (StringRef Part : Parts) {
1348 OS << Sep << Part;
1349 Sep = Separator;
1350 }
1351 return std::string(OS.str());
1352}
1353
1354static llvm::Function *
1355emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1356 const Expr *CombinerInitializer, const VarDecl *In,
1357 const VarDecl *Out, bool IsCombiner) {
1358 // void .omp_combiner.(Ty *in, Ty *out);
1359 ASTContext &C = CGM.getContext();
1360 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1361 FunctionArgList Args;
1362 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1363 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1364 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1365 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1366 Args.push_back(&OmpOutParm);
1367 Args.push_back(&OmpInParm);
1368 const CGFunctionInfo &FnInfo =
1369 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1370 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1371 std::string Name = CGM.getOpenMPRuntime().getName(
1372 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1373 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1374 Name, &CGM.getModule());
1375 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1376 if (CGM.getLangOpts().Optimize) {
1377 Fn->removeFnAttr(llvm::Attribute::NoInline);
1378 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1379 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1380 }
1381 CodeGenFunction CGF(CGM);
1382 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1383 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1384 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1385 Out->getLocation());
1386 CodeGenFunction::OMPPrivateScope Scope(CGF);
1387 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1388 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1389 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1390 .getAddress(CGF);
1391 });
1392 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1393 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1394 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1395 .getAddress(CGF);
1396 });
1397 (void)Scope.Privatize();
1398 if (!IsCombiner && Out->hasInit() &&
1399 !CGF.isTrivialInitializer(Out->getInit())) {
1400 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1401 Out->getType().getQualifiers(),
1402 /*IsInitializer=*/true);
1403 }
1404 if (CombinerInitializer)
1405 CGF.EmitIgnoredExpr(CombinerInitializer);
1406 Scope.ForceCleanup();
1407 CGF.FinishFunction();
1408 return Fn;
1409}
1410
1411void CGOpenMPRuntime::emitUserDefinedReduction(
1412 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1413 if (UDRMap.count(D) > 0)
1414 return;
1415 llvm::Function *Combiner = emitCombinerOrInitializer(
1416 CGM, D->getType(), D->getCombiner(),
1417 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1418 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1419 /*IsCombiner=*/true);
1420 llvm::Function *Initializer = nullptr;
1421 if (const Expr *Init = D->getInitializer()) {
1422 Initializer = emitCombinerOrInitializer(
1423 CGM, D->getType(),
1424 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1425 : nullptr,
1426 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1427 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1428 /*IsCombiner=*/false);
1429 }
1430 UDRMap.try_emplace(D, Combiner, Initializer);
1431 if (CGF) {
1432 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1433 Decls.second.push_back(D);
1434 }
1435}
1436
1437std::pair<llvm::Function *, llvm::Function *>
1438CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1439 auto I = UDRMap.find(D);
1440 if (I != UDRMap.end())
1441 return I->second;
1442 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1443 return UDRMap.lookup(D);
1444}
1445
1446namespace {
1447// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1448// Builder if one is present.
1449struct PushAndPopStackRAII {
1450 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1451 bool HasCancel)
1452 : OMPBuilder(OMPBuilder) {
1453 if (!OMPBuilder)
1454 return;
1455
1456 // The following callback is the crucial part of clangs cleanup process.
1457 //
1458 // NOTE:
1459 // Once the OpenMPIRBuilder is used to create parallel regions (and
1460 // similar), the cancellation destination (Dest below) is determined via
1461 // IP. That means if we have variables to finalize we split the block at IP,
1462 // use the new block (=BB) as destination to build a JumpDest (via
1463 // getJumpDestInCurrentScope(BB)) which then is fed to
1464 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1465 // to push & pop an FinalizationInfo object.
1466 // The FiniCB will still be needed but at the point where the
1467 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1468 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1469 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1470, __PRETTY_FUNCTION__))
1470 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1470, __PRETTY_FUNCTION__))
;
1471 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1472 CGF.Builder.restoreIP(IP);
1473 CodeGenFunction::JumpDest Dest =
1474 CGF.getOMPCancelDestination(OMPD_parallel);
1475 CGF.EmitBranchThroughCleanup(Dest);
1476 };
1477
1478 // TODO: Remove this once we emit parallel regions through the
1479 // OpenMPIRBuilder as it can do this setup internally.
1480 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1481 {FiniCB, OMPD_parallel, HasCancel});
1482 OMPBuilder->pushFinalizationCB(std::move(FI));
1483 }
1484 ~PushAndPopStackRAII() {
1485 if (OMPBuilder)
1486 OMPBuilder->popFinalizationCB();
1487 }
1488 llvm::OpenMPIRBuilder *OMPBuilder;
1489};
1490} // namespace
1491
1492static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1493 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1494 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1495 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1496 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1497, __PRETTY_FUNCTION__))
1497 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1497, __PRETTY_FUNCTION__))
;
1498 CodeGenFunction CGF(CGM, true);
1499 bool HasCancel = false;
1500 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1501 HasCancel = OPD->hasCancel();
1502 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1503 HasCancel = OPSD->hasCancel();
1504 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1505 HasCancel = OPFD->hasCancel();
1506 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1507 HasCancel = OPFD->hasCancel();
1508 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1509 HasCancel = OPFD->hasCancel();
1510 else if (const auto *OPFD =
1511 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1512 HasCancel = OPFD->hasCancel();
1513 else if (const auto *OPFD =
1514 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1515 HasCancel = OPFD->hasCancel();
1516
1517 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1518 // parallel region to make cancellation barriers work properly.
1519 llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder();
1520 PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
1521 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1522 HasCancel, OutlinedHelperName);
1523 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1524 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1525}
1526
1527llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1528 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1529 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1530 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1531 return emitParallelOrTeamsOutlinedFunction(
1532 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1533}
1534
1535llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1536 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1537 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1538 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1539 return emitParallelOrTeamsOutlinedFunction(
1540 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1541}
1542
1543llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1544 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1545 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1546 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1547 bool Tied, unsigned &NumberOfParts) {
1548 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1549 PrePostActionTy &) {
1550 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1551 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1552 llvm::Value *TaskArgs[] = {
1553 UpLoc, ThreadID,
1554 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1555 TaskTVar->getType()->castAs<PointerType>())
1556 .getPointer(CGF)};
1557 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1558 };
1559 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1560 UntiedCodeGen);
1561 CodeGen.setAction(Action);
1562 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1563, __PRETTY_FUNCTION__))
1563 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1563, __PRETTY_FUNCTION__))
;
1564 const OpenMPDirectiveKind Region =
1565 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1566 : OMPD_task;
1567 const CapturedStmt *CS = D.getCapturedStmt(Region);
1568 bool HasCancel = false;
1569 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1570 HasCancel = TD->hasCancel();
1571 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1572 HasCancel = TD->hasCancel();
1573 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1574 HasCancel = TD->hasCancel();
1575 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1576 HasCancel = TD->hasCancel();
1577
1578 CodeGenFunction CGF(CGM, true);
1579 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1580 InnermostKind, HasCancel, Action);
1581 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1582 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1583 if (!Tied)
1584 NumberOfParts = Action.getNumberOfParts();
1585 return Res;
1586}
1587
1588static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1589 const RecordDecl *RD, const CGRecordLayout &RL,
1590 ArrayRef<llvm::Constant *> Data) {
1591 llvm::StructType *StructTy = RL.getLLVMType();
1592 unsigned PrevIdx = 0;
1593 ConstantInitBuilder CIBuilder(CGM);
1594 auto DI = Data.begin();
1595 for (const FieldDecl *FD : RD->fields()) {
1596 unsigned Idx = RL.getLLVMFieldNo(FD);
1597 // Fill the alignment.
1598 for (unsigned I = PrevIdx; I < Idx; ++I)
1599 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1600 PrevIdx = Idx + 1;
1601 Fields.add(*DI);
1602 ++DI;
1603 }
1604}
1605
1606template <class... As>
1607static llvm::GlobalVariable *
1608createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1609 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1610 As &&... Args) {
1611 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1612 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1613 ConstantInitBuilder CIBuilder(CGM);
1614 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1615 buildStructValue(Fields, CGM, RD, RL, Data);
1616 return Fields.finishAndCreateGlobal(
1617 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1618 std::forward<As>(Args)...);
1619}
1620
1621template <typename T>
1622static void
1623createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1624 ArrayRef<llvm::Constant *> Data,
1625 T &Parent) {
1626 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1627 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1628 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1629 buildStructValue(Fields, CGM, RD, RL, Data);
1630 Fields.finishAndAddTo(Parent);
1631}
1632
1633Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1634 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1635 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1636 FlagsTy FlagsKey(Flags, Reserved2Flags);
1637 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1638 if (!Entry) {
1639 if (!DefaultOpenMPPSource) {
1640 // Initialize default location for psource field of ident_t structure of
1641 // all ident_t objects. Format is ";file;function;line;column;;".
1642 // Taken from
1643 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1644 DefaultOpenMPPSource =
1645 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1646 DefaultOpenMPPSource =
1647 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1648 }
1649
1650 llvm::Constant *Data[] = {
1651 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1652 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1653 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1654 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1655 llvm::GlobalValue *DefaultOpenMPLocation =
1656 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1657 llvm::GlobalValue::PrivateLinkage);
1658 DefaultOpenMPLocation->setUnnamedAddr(
1659 llvm::GlobalValue::UnnamedAddr::Global);
1660
1661 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1662 }
1663 return Address(Entry, Align);
1664}
1665
1666void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1667 bool AtCurrentPoint) {
1668 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1669 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1669, __PRETTY_FUNCTION__))
;
1670
1671 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1672 if (AtCurrentPoint) {
1673 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1674 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1675 } else {
1676 Elem.second.ServiceInsertPt =
1677 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1678 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1679 }
1680}
1681
1682void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1683 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1684 if (Elem.second.ServiceInsertPt) {
1685 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1686 Elem.second.ServiceInsertPt = nullptr;
1687 Ptr->eraseFromParent();
1688 }
1689}
1690
1691llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1692 SourceLocation Loc,
1693 unsigned Flags) {
1694 Flags |= OMP_IDENT_KMPC;
1695 // If no debug info is generated - return global default location.
1696 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1697 Loc.isInvalid())
1698 return getOrCreateDefaultLocation(Flags).getPointer();
1699
1700 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1700, __PRETTY_FUNCTION__))
;
1701
1702 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1703 Address LocValue = Address::invalid();
1704 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1705 if (I != OpenMPLocThreadIDMap.end())
1706 LocValue = Address(I->second.DebugLoc, Align);
1707
1708 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1709 // GetOpenMPThreadID was called before this routine.
1710 if (!LocValue.isValid()) {
1711 // Generate "ident_t .kmpc_loc.addr;"
1712 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1713 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1714 Elem.second.DebugLoc = AI.getPointer();
1715 LocValue = AI;
1716
1717 if (!Elem.second.ServiceInsertPt)
1718 setLocThreadIdInsertPt(CGF);
1719 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1720 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1721 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1722 CGF.getTypeSize(IdentQTy));
1723 }
1724
1725 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1726 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1727 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1728 LValue PSource =
1729 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1730
1731 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1732 if (OMPDebugLoc == nullptr) {
1733 SmallString<128> Buffer2;
1734 llvm::raw_svector_ostream OS2(Buffer2);
1735 // Build debug location
1736 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1737 OS2 << ";" << PLoc.getFilename() << ";";
1738 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1739 OS2 << FD->getQualifiedNameAsString();
1740 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1741 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1742 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1743 }
1744 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1745 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1746
1747 // Our callers always pass this to a runtime function, so for
1748 // convenience, go ahead and return a naked pointer.
1749 return LocValue.getPointer();
1750}
1751
1752llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1753 SourceLocation Loc) {
1754 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1754, __PRETTY_FUNCTION__))
;
1755
1756 llvm::Value *ThreadID = nullptr;
1757 // Check whether we've already cached a load of the thread id in this
1758 // function.
1759 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1760 if (I != OpenMPLocThreadIDMap.end()) {
1761 ThreadID = I->second.ThreadID;
1762 if (ThreadID != nullptr)
1763 return ThreadID;
1764 }
1765 // If exceptions are enabled, do not use parameter to avoid possible crash.
1766 if (auto *OMPRegionInfo =
1767 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1768 if (OMPRegionInfo->getThreadIDVariable()) {
1769 // Check if this an outlined function with thread id passed as argument.
1770 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1771 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1772 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1773 !CGF.getLangOpts().CXXExceptions ||
1774 CGF.Builder.GetInsertBlock() == TopBlock ||
1775 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1776 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1777 TopBlock ||
1778 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1779 CGF.Builder.GetInsertBlock()) {
1780 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1781 // If value loaded in entry block, cache it and use it everywhere in
1782 // function.
1783 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1784 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1785 Elem.second.ThreadID = ThreadID;
1786 }
1787 return ThreadID;
1788 }
1789 }
1790 }
1791
1792 // This is not an outlined function region - need to call __kmpc_int32
1793 // kmpc_global_thread_num(ident_t *loc).
1794 // Generate thread id value and cache this value for use across the
1795 // function.
1796 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1797 if (!Elem.second.ServiceInsertPt)
1798 setLocThreadIdInsertPt(CGF);
1799 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1800 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1801 llvm::CallInst *Call = CGF.Builder.CreateCall(
1802 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1803 emitUpdateLocation(CGF, Loc));
1804 Call->setCallingConv(CGF.getRuntimeCC());
1805 Elem.second.ThreadID = Call;
1806 return Call;
1807}
1808
1809void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1810 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1810, __PRETTY_FUNCTION__))
;
1811 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1812 clearLocThreadIdInsertPt(CGF);
1813 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1814 }
1815 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1816 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1817 UDRMap.erase(D);
1818 FunctionUDRMap.erase(CGF.CurFn);
1819 }
1820 auto I = FunctionUDMMap.find(CGF.CurFn);
1821 if (I != FunctionUDMMap.end()) {
1822 for(const auto *D : I->second)
1823 UDMMap.erase(D);
1824 FunctionUDMMap.erase(I);
1825 }
1826 LastprivateConditionalToTypes.erase(CGF.CurFn);
1827}
1828
1829llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1830 return IdentTy->getPointerTo();
1831}
1832
1833llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1834 if (!Kmpc_MicroTy) {
1835 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1836 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1837 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1838 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1839 }
1840 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1841}
1842
1843llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1844 llvm::FunctionCallee RTLFn = nullptr;
1845 switch (static_cast<OpenMPRTLFunction>(Function)) {
1846 case OMPRTL__kmpc_fork_call: {
1847 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1848 // microtask, ...);
1849 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1850 getKmpc_MicroPointerTy()};
1851 auto *FnTy =
1852 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1853 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1854 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1855 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1856 llvm::LLVMContext &Ctx = F->getContext();
1857 llvm::MDBuilder MDB(Ctx);
1858 // Annotate the callback behavior of the __kmpc_fork_call:
1859 // - The callback callee is argument number 2 (microtask).
1860 // - The first two arguments of the callback callee are unknown (-1).
1861 // - All variadic arguments to the __kmpc_fork_call are passed to the
1862 // callback callee.
1863 F->addMetadata(
1864 llvm::LLVMContext::MD_callback,
1865 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1866 2, {-1, -1},
1867 /* VarArgsArePassed */ true)}));
1868 }
1869 }
1870 break;
1871 }
1872 case OMPRTL__kmpc_global_thread_num: {
1873 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1874 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1875 auto *FnTy =
1876 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1877 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1878 break;
1879 }
1880 case OMPRTL__kmpc_threadprivate_cached: {
1881 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1882 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1883 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1884 CGM.VoidPtrTy, CGM.SizeTy,
1885 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1886 auto *FnTy =
1887 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1888 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1889 break;
1890 }
1891 case OMPRTL__kmpc_critical: {
1892 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1893 // kmp_critical_name *crit);
1894 llvm::Type *TypeParams[] = {
1895 getIdentTyPointerTy(), CGM.Int32Ty,
1896 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1897 auto *FnTy =
1898 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1899 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1900 break;
1901 }
1902 case OMPRTL__kmpc_critical_with_hint: {
1903 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1904 // kmp_critical_name *crit, uintptr_t hint);
1905 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1906 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1907 CGM.IntPtrTy};
1908 auto *FnTy =
1909 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1910 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1911 break;
1912 }
1913 case OMPRTL__kmpc_threadprivate_register: {
1914 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1915 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1916 // typedef void *(*kmpc_ctor)(void *);
1917 auto *KmpcCtorTy =
1918 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1919 /*isVarArg*/ false)->getPointerTo();
1920 // typedef void *(*kmpc_cctor)(void *, void *);
1921 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1922 auto *KmpcCopyCtorTy =
1923 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1924 /*isVarArg*/ false)
1925 ->getPointerTo();
1926 // typedef void (*kmpc_dtor)(void *);
1927 auto *KmpcDtorTy =
1928 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1929 ->getPointerTo();
1930 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1931 KmpcCopyCtorTy, KmpcDtorTy};
1932 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1933 /*isVarArg*/ false);
1934 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1935 break;
1936 }
1937 case OMPRTL__kmpc_end_critical: {
1938 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1939 // kmp_critical_name *crit);
1940 llvm::Type *TypeParams[] = {
1941 getIdentTyPointerTy(), CGM.Int32Ty,
1942 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1943 auto *FnTy =
1944 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1945 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1946 break;
1947 }
1948 case OMPRTL__kmpc_cancel_barrier: {
1949 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1950 // global_tid);
1951 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1952 auto *FnTy =
1953 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1954 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1955 break;
1956 }
1957 case OMPRTL__kmpc_barrier: {
1958 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1959 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1960 auto *FnTy =
1961 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1962 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1963 break;
1964 }
1965 case OMPRTL__kmpc_for_static_fini: {
1966 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1967 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1968 auto *FnTy =
1969 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1970 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1971 break;
1972 }
1973 case OMPRTL__kmpc_push_num_threads: {
1974 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1975 // kmp_int32 num_threads)
1976 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1977 CGM.Int32Ty};
1978 auto *FnTy =
1979 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1980 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1981 break;
1982 }
1983 case OMPRTL__kmpc_serialized_parallel: {
1984 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1985 // global_tid);
1986 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1987 auto *FnTy =
1988 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1989 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1990 break;
1991 }
1992 case OMPRTL__kmpc_end_serialized_parallel: {
1993 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1994 // global_tid);
1995 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1996 auto *FnTy =
1997 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1998 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1999 break;
2000 }
2001 case OMPRTL__kmpc_flush: {
2002 // Build void __kmpc_flush(ident_t *loc);
2003 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
2004 auto *FnTy =
2005 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2006 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
2007 break;
2008 }
2009 case OMPRTL__kmpc_master: {
2010 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
2011 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2012 auto *FnTy =
2013 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2014 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
2015 break;
2016 }
2017 case OMPRTL__kmpc_end_master: {
2018 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
2019 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2020 auto *FnTy =
2021 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2022 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
2023 break;
2024 }
2025 case OMPRTL__kmpc_omp_taskyield: {
2026 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
2027 // int end_part);
2028 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2029 auto *FnTy =
2030 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2031 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
2032 break;
2033 }
2034 case OMPRTL__kmpc_single: {
2035 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
2036 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2037 auto *FnTy =
2038 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2039 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
2040 break;
2041 }
2042 case OMPRTL__kmpc_end_single: {
2043 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
2044 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2045 auto *FnTy =
2046 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2047 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
2048 break;
2049 }
2050 case OMPRTL__kmpc_omp_task_alloc: {
2051 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
2052 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2053 // kmp_routine_entry_t *task_entry);
2054 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2055, __PRETTY_FUNCTION__))
2055 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2055, __PRETTY_FUNCTION__))
;
2056 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2057 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
2058 // Return void * and then cast to particular kmp_task_t type.
2059 auto *FnTy =
2060 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2061 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
2062 break;
2063 }
2064 case OMPRTL__kmpc_omp_target_task_alloc: {
2065 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
2066 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2067 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
2068 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2069, __PRETTY_FUNCTION__))
2069 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2069, __PRETTY_FUNCTION__))
;
2070 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2071 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
2072 CGM.Int64Ty};
2073 // Return void * and then cast to particular kmp_task_t type.
2074 auto *FnTy =
2075 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2076 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2077 break;
2078 }
2079 case OMPRTL__kmpc_omp_task: {
2080 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2081 // *new_task);
2082 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2083 CGM.VoidPtrTy};
2084 auto *FnTy =
2085 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2086 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2087 break;
2088 }
2089 case OMPRTL__kmpc_copyprivate: {
2090 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2091 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2092 // kmp_int32 didit);
2093 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2094 auto *CpyFnTy =
2095 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2096 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2097 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2098 CGM.Int32Ty};
2099 auto *FnTy =
2100 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2101 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2102 break;
2103 }
2104 case OMPRTL__kmpc_reduce: {
2105 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2106 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2107 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2108 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2109 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2110 /*isVarArg=*/false);
2111 llvm::Type *TypeParams[] = {
2112 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2113 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2114 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2115 auto *FnTy =
2116 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2117 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2118 break;
2119 }
2120 case OMPRTL__kmpc_reduce_nowait: {
2121 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2122 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2123 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2124 // *lck);
2125 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2126 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2127 /*isVarArg=*/false);
2128 llvm::Type *TypeParams[] = {
2129 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2130 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2131 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2132 auto *FnTy =
2133 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2134 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2135 break;
2136 }
2137 case OMPRTL__kmpc_end_reduce: {
2138 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2139 // kmp_critical_name *lck);
2140 llvm::Type *TypeParams[] = {
2141 getIdentTyPointerTy(), CGM.Int32Ty,
2142 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2143 auto *FnTy =
2144 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2145 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2146 break;
2147 }
2148 case OMPRTL__kmpc_end_reduce_nowait: {
2149 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2150 // kmp_critical_name *lck);
2151 llvm::Type *TypeParams[] = {
2152 getIdentTyPointerTy(), CGM.Int32Ty,
2153 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2154 auto *FnTy =
2155 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2156 RTLFn =
2157 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2158 break;
2159 }
2160 case OMPRTL__kmpc_omp_task_begin_if0: {
2161 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2162 // *new_task);
2163 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2164 CGM.VoidPtrTy};
2165 auto *FnTy =
2166 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2167 RTLFn =
2168 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2169 break;
2170 }
2171 case OMPRTL__kmpc_omp_task_complete_if0: {
2172 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2173 // *new_task);
2174 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2175 CGM.VoidPtrTy};
2176 auto *FnTy =
2177 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2178 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2179 /*Name=*/"__kmpc_omp_task_complete_if0");
2180 break;
2181 }
2182 case OMPRTL__kmpc_ordered: {
2183 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2184 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2185 auto *FnTy =
2186 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2187 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2188 break;
2189 }
2190 case OMPRTL__kmpc_end_ordered: {
2191 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2192 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2193 auto *FnTy =
2194 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2195 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2196 break;
2197 }
2198 case OMPRTL__kmpc_omp_taskwait: {
2199 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2200 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2201 auto *FnTy =
2202 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2203 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2204 break;
2205 }
2206 case OMPRTL__kmpc_taskgroup: {
2207 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2208 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2209 auto *FnTy =
2210 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2211 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2212 break;
2213 }
2214 case OMPRTL__kmpc_end_taskgroup: {
2215 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2216 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2217 auto *FnTy =
2218 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2219 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2220 break;
2221 }
2222 case OMPRTL__kmpc_push_proc_bind: {
2223 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2224 // int proc_bind)
2225 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2226 auto *FnTy =
2227 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2228 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2229 break;
2230 }
2231 case OMPRTL__kmpc_omp_task_with_deps: {
2232 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2233 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2234 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2235 llvm::Type *TypeParams[] = {
2236 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2237 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2238 auto *FnTy =
2239 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2240 RTLFn =
2241 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2242 break;
2243 }
2244 case OMPRTL__kmpc_omp_wait_deps: {
2245 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2246 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2247 // kmp_depend_info_t *noalias_dep_list);
2248 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2249 CGM.Int32Ty, CGM.VoidPtrTy,
2250 CGM.Int32Ty, CGM.VoidPtrTy};
2251 auto *FnTy =
2252 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2253 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2254 break;
2255 }
2256 case OMPRTL__kmpc_cancellationpoint: {
2257 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2258 // global_tid, kmp_int32 cncl_kind)
2259 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2260 auto *FnTy =
2261 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2262 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2263 break;
2264 }
2265 case OMPRTL__kmpc_cancel: {
2266 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2267 // kmp_int32 cncl_kind)
2268 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2269 auto *FnTy =
2270 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2271 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2272 break;
2273 }
2274 case OMPRTL__kmpc_push_num_teams: {
2275 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2276 // kmp_int32 num_teams, kmp_int32 num_threads)
2277 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2278 CGM.Int32Ty};
2279 auto *FnTy =
2280 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2281 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2282 break;
2283 }
2284 case OMPRTL__kmpc_fork_teams: {
2285 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2286 // microtask, ...);
2287 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2288 getKmpc_MicroPointerTy()};
2289 auto *FnTy =
2290 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2291 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2292 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2293 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2294 llvm::LLVMContext &Ctx = F->getContext();
2295 llvm::MDBuilder MDB(Ctx);
2296 // Annotate the callback behavior of the __kmpc_fork_teams:
2297 // - The callback callee is argument number 2 (microtask).
2298 // - The first two arguments of the callback callee are unknown (-1).
2299 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2300 // callback callee.
2301 F->addMetadata(
2302 llvm::LLVMContext::MD_callback,
2303 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2304 2, {-1, -1},
2305 /* VarArgsArePassed */ true)}));
2306 }
2307 }
2308 break;
2309 }
2310 case OMPRTL__kmpc_taskloop: {
2311 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2312 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2313 // sched, kmp_uint64 grainsize, void *task_dup);
2314 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2315 CGM.IntTy,
2316 CGM.VoidPtrTy,
2317 CGM.IntTy,
2318 CGM.Int64Ty->getPointerTo(),
2319 CGM.Int64Ty->getPointerTo(),
2320 CGM.Int64Ty,
2321 CGM.IntTy,
2322 CGM.IntTy,
2323 CGM.Int64Ty,
2324 CGM.VoidPtrTy};
2325 auto *FnTy =
2326 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2327 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2328 break;
2329 }
2330 case OMPRTL__kmpc_doacross_init: {
2331 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2332 // num_dims, struct kmp_dim *dims);
2333 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2334 CGM.Int32Ty,
2335 CGM.Int32Ty,
2336 CGM.VoidPtrTy};
2337 auto *FnTy =
2338 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2339 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2340 break;
2341 }
2342 case OMPRTL__kmpc_doacross_fini: {
2343 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2344 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2345 auto *FnTy =
2346 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2347 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2348 break;
2349 }
2350 case OMPRTL__kmpc_doacross_post: {
2351 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2352 // *vec);
2353 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2354 CGM.Int64Ty->getPointerTo()};
2355 auto *FnTy =
2356 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2357 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2358 break;
2359 }
2360 case OMPRTL__kmpc_doacross_wait: {
2361 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2362 // *vec);
2363 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2364 CGM.Int64Ty->getPointerTo()};
2365 auto *FnTy =
2366 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2367 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2368 break;
2369 }
2370 case OMPRTL__kmpc_task_reduction_init: {
2371 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2372 // *data);
2373 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2374 auto *FnTy =
2375 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2376 RTLFn =
2377 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2378 break;
2379 }
2380 case OMPRTL__kmpc_task_reduction_get_th_data: {
2381 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2382 // *d);
2383 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2384 auto *FnTy =
2385 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2386 RTLFn = CGM.CreateRuntimeFunction(
2387 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2388 break;
2389 }
2390 case OMPRTL__kmpc_alloc: {
2391 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2392 // al); omp_allocator_handle_t type is void *.
2393 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2394 auto *FnTy =
2395 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2396 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2397 break;
2398 }
2399 case OMPRTL__kmpc_free: {
2400 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2401 // al); omp_allocator_handle_t type is void *.
2402 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2403 auto *FnTy =
2404 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2405 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2406 break;
2407 }
2408 case OMPRTL__kmpc_push_target_tripcount: {
2409 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2410 // size);
2411 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2412 llvm::FunctionType *FnTy =
2413 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2414 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2415 break;
2416 }
2417 case OMPRTL__tgt_target: {
2418 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2419 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2420 // *arg_types);
2421 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2422 CGM.VoidPtrTy,
2423 CGM.Int32Ty,
2424 CGM.VoidPtrPtrTy,
2425 CGM.VoidPtrPtrTy,
2426 CGM.Int64Ty->getPointerTo(),
2427 CGM.Int64Ty->getPointerTo()};
2428 auto *FnTy =
2429 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2430 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2431 break;
2432 }
2433 case OMPRTL__tgt_target_nowait: {
2434 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2435 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2436 // int64_t *arg_types);
2437 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2438 CGM.VoidPtrTy,
2439 CGM.Int32Ty,
2440 CGM.VoidPtrPtrTy,
2441 CGM.VoidPtrPtrTy,
2442 CGM.Int64Ty->getPointerTo(),
2443 CGM.Int64Ty->getPointerTo()};
2444 auto *FnTy =
2445 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2446 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2447 break;
2448 }
2449 case OMPRTL__tgt_target_teams: {
2450 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2451 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2452 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2453 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2454 CGM.VoidPtrTy,
2455 CGM.Int32Ty,
2456 CGM.VoidPtrPtrTy,
2457 CGM.VoidPtrPtrTy,
2458 CGM.Int64Ty->getPointerTo(),
2459 CGM.Int64Ty->getPointerTo(),
2460 CGM.Int32Ty,
2461 CGM.Int32Ty};
2462 auto *FnTy =
2463 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2464 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2465 break;
2466 }
2467 case OMPRTL__tgt_target_teams_nowait: {
2468 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2469 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2470 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2471 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2472 CGM.VoidPtrTy,
2473 CGM.Int32Ty,
2474 CGM.VoidPtrPtrTy,
2475 CGM.VoidPtrPtrTy,
2476 CGM.Int64Ty->getPointerTo(),
2477 CGM.Int64Ty->getPointerTo(),
2478 CGM.Int32Ty,
2479 CGM.Int32Ty};
2480 auto *FnTy =
2481 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2482 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2483 break;
2484 }
2485 case OMPRTL__tgt_register_requires: {
2486 // Build void __tgt_register_requires(int64_t flags);
2487 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2488 auto *FnTy =
2489 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2490 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2491 break;
2492 }
2493 case OMPRTL__tgt_target_data_begin: {
2494 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2495 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2496 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2497 CGM.Int32Ty,
2498 CGM.VoidPtrPtrTy,
2499 CGM.VoidPtrPtrTy,
2500 CGM.Int64Ty->getPointerTo(),
2501 CGM.Int64Ty->getPointerTo()};
2502 auto *FnTy =
2503 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2504 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2505 break;
2506 }
2507 case OMPRTL__tgt_target_data_begin_nowait: {
2508 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2509 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2510 // *arg_types);
2511 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2512 CGM.Int32Ty,
2513 CGM.VoidPtrPtrTy,
2514 CGM.VoidPtrPtrTy,
2515 CGM.Int64Ty->getPointerTo(),
2516 CGM.Int64Ty->getPointerTo()};
2517 auto *FnTy =
2518 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2519 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2520 break;
2521 }
2522 case OMPRTL__tgt_target_data_end: {
2523 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2524 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2525 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2526 CGM.Int32Ty,
2527 CGM.VoidPtrPtrTy,
2528 CGM.VoidPtrPtrTy,
2529 CGM.Int64Ty->getPointerTo(),
2530 CGM.Int64Ty->getPointerTo()};
2531 auto *FnTy =
2532 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2533 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2534 break;
2535 }
2536 case OMPRTL__tgt_target_data_end_nowait: {
2537 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2538 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2539 // *arg_types);
2540 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2541 CGM.Int32Ty,
2542 CGM.VoidPtrPtrTy,
2543 CGM.VoidPtrPtrTy,
2544 CGM.Int64Ty->getPointerTo(),
2545 CGM.Int64Ty->getPointerTo()};
2546 auto *FnTy =
2547 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2548 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2549 break;
2550 }
2551 case OMPRTL__tgt_target_data_update: {
2552 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2553 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2554 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2555 CGM.Int32Ty,
2556 CGM.VoidPtrPtrTy,
2557 CGM.VoidPtrPtrTy,
2558 CGM.Int64Ty->getPointerTo(),
2559 CGM.Int64Ty->getPointerTo()};
2560 auto *FnTy =
2561 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2562 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2563 break;
2564 }
2565 case OMPRTL__tgt_target_data_update_nowait: {
2566 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2567 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2568 // *arg_types);
2569 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2570 CGM.Int32Ty,
2571 CGM.VoidPtrPtrTy,
2572 CGM.VoidPtrPtrTy,
2573 CGM.Int64Ty->getPointerTo(),
2574 CGM.Int64Ty->getPointerTo()};
2575 auto *FnTy =
2576 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2577 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2578 break;
2579 }
2580 case OMPRTL__tgt_mapper_num_components: {
2581 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2582 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2583 auto *FnTy =
2584 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2585 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2586 break;
2587 }
2588 case OMPRTL__tgt_push_mapper_component: {
2589 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2590 // *base, void *begin, int64_t size, int64_t type);
2591 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2592 CGM.Int64Ty, CGM.Int64Ty};
2593 auto *FnTy =
2594 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2595 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2596 break;
2597 }
2598 }
2599 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2599, __PRETTY_FUNCTION__))
;
2600 return RTLFn;
2601}
2602
2603llvm::FunctionCallee
2604CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2605 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2606, __PRETTY_FUNCTION__))
2606 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2606, __PRETTY_FUNCTION__))
;
2607 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2608 : "__kmpc_for_static_init_4u")
2609 : (IVSigned ? "__kmpc_for_static_init_8"
2610 : "__kmpc_for_static_init_8u");
2611 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2612 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2613 llvm::Type *TypeParams[] = {
2614 getIdentTyPointerTy(), // loc
2615 CGM.Int32Ty, // tid
2616 CGM.Int32Ty, // schedtype
2617 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2618 PtrTy, // p_lower
2619 PtrTy, // p_upper
2620 PtrTy, // p_stride
2621 ITy, // incr
2622 ITy // chunk
2623 };
2624 auto *FnTy =
2625 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2626 return CGM.CreateRuntimeFunction(FnTy, Name);
2627}
2628
2629llvm::FunctionCallee
2630CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2631 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2632, __PRETTY_FUNCTION__))
2632 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2632, __PRETTY_FUNCTION__))
;
2633 StringRef Name =
2634 IVSize == 32
2635 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2636 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2637 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2638 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2639 CGM.Int32Ty, // tid
2640 CGM.Int32Ty, // schedtype
2641 ITy, // lower
2642 ITy, // upper
2643 ITy, // stride
2644 ITy // chunk
2645 };
2646 auto *FnTy =
2647 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2648 return CGM.CreateRuntimeFunction(FnTy, Name);
2649}
2650
2651llvm::FunctionCallee
2652CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2653 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2654, __PRETTY_FUNCTION__))
2654 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2654, __PRETTY_FUNCTION__))
;
2655 StringRef Name =
2656 IVSize == 32
2657 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2658 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2659 llvm::Type *TypeParams[] = {
2660 getIdentTyPointerTy(), // loc
2661 CGM.Int32Ty, // tid
2662 };
2663 auto *FnTy =
2664 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2665 return CGM.CreateRuntimeFunction(FnTy, Name);
2666}
2667
2668llvm::FunctionCallee
2669CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2670 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2671, __PRETTY_FUNCTION__))
2671 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2671, __PRETTY_FUNCTION__))
;
2672 StringRef Name =
2673 IVSize == 32
2674 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2675 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2676 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2677 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2678 llvm::Type *TypeParams[] = {
2679 getIdentTyPointerTy(), // loc
2680 CGM.Int32Ty, // tid
2681 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2682 PtrTy, // p_lower
2683 PtrTy, // p_upper
2684 PtrTy // p_stride
2685 };
2686 auto *FnTy =
2687 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2688 return CGM.CreateRuntimeFunction(FnTy, Name);
2689}
2690
2691/// Obtain information that uniquely identifies a target entry. This
2692/// consists of the file and device IDs as well as line number associated with
2693/// the relevant entry source location.
2694static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2695 unsigned &DeviceID, unsigned &FileID,
2696 unsigned &LineNum) {
2697 SourceManager &SM = C.getSourceManager();
2698
2699 // The loc should be always valid and have a file ID (the user cannot use
2700 // #pragma directives in macros)
2701
2702 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2702, __PRETTY_FUNCTION__))
;
2703
2704 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2705 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2705, __PRETTY_FUNCTION__))
;
2706
2707 llvm::sys::fs::UniqueID ID;
2708 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2709 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2710 << PLoc.getFilename() << EC.message();
2711
2712 DeviceID = ID.getDevice();
2713 FileID = ID.getFile();
2714 LineNum = PLoc.getLine();
2715}
2716
2717Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2718 if (CGM.getLangOpts().OpenMPSimd)
2719 return Address::invalid();
2720 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2721 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2722 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2723 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2724 HasRequiresUnifiedSharedMemory))) {
2725 SmallString<64> PtrName;
2726 {
2727 llvm::raw_svector_ostream OS(PtrName);
2728 OS << CGM.getMangledName(GlobalDecl(VD));
2729 if (!VD->isExternallyVisible()) {
2730 unsigned DeviceID, FileID, Line;
2731 getTargetEntryUniqueInfo(CGM.getContext(),
2732 VD->getCanonicalDecl()->getBeginLoc(),
2733 DeviceID, FileID, Line);
2734 OS << llvm::format("_%x", FileID);
2735 }
2736 OS << "_decl_tgt_ref_ptr";
2737 }
2738 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2739 if (!Ptr) {
2740 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2741 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2742 PtrName);
2743
2744 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2745 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2746
2747 if (!CGM.getLangOpts().OpenMPIsDevice)
2748 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2749 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2750 }
2751 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2752 }
2753 return Address::invalid();
2754}
2755
2756llvm::Constant *
2757CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2758 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2759, __PRETTY_FUNCTION__))
2759 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2759, __PRETTY_FUNCTION__))
;
2760 // Lookup the entry, lazily creating it if necessary.
2761 std::string Suffix = getName({"cache", ""});
2762 return getOrCreateInternalVariable(
2763 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2764}
2765
2766Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2767 const VarDecl *VD,
2768 Address VDAddr,
2769 SourceLocation Loc) {
2770 if (CGM.getLangOpts().OpenMPUseTLS &&
2771 CGM.getContext().getTargetInfo().isTLSSupported())
2772 return VDAddr;
2773
2774 llvm::Type *VarTy = VDAddr.getElementType();
2775 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2776 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2777 CGM.Int8PtrTy),
2778 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2779 getOrCreateThreadPrivateCache(VD)};
2780 return Address(CGF.EmitRuntimeCall(
2781 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2782 VDAddr.getAlignment());
2783}
2784
2785void CGOpenMPRuntime::emitThreadPrivateVarInit(
2786 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2787 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2788 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2789 // library.
2790 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2791 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2792 OMPLoc);
2793 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2794 // to register constructor/destructor for variable.
2795 llvm::Value *Args[] = {
2796 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2797 Ctor, CopyCtor, Dtor};
2798 CGF.EmitRuntimeCall(
2799 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2800}
2801
2802llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2803 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2804 bool PerformInit, CodeGenFunction *CGF) {
2805 if (CGM.getLangOpts().OpenMPUseTLS &&
2806 CGM.getContext().getTargetInfo().isTLSSupported())
2807 return nullptr;
2808
2809 VD = VD->getDefinition(CGM.getContext());
2810 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2811 QualType ASTTy = VD->getType();
2812
2813 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2814 const Expr *Init = VD->getAnyInitializer();
2815 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2816 // Generate function that re-emits the declaration's initializer into the
2817 // threadprivate copy of the variable VD
2818 CodeGenFunction CtorCGF(CGM);
2819 FunctionArgList Args;
2820 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2821 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2822 ImplicitParamDecl::Other);
2823 Args.push_back(&Dst);
2824
2825 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2826 CGM.getContext().VoidPtrTy, Args);
2827 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2828 std::string Name = getName({"__kmpc_global_ctor_", ""});
2829 llvm::Function *Fn =
2830 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2831 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2832 Args, Loc, Loc);
2833 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2834 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2835 CGM.getContext().VoidPtrTy, Dst.getLocation());
2836 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2837 Arg = CtorCGF.Builder.CreateElementBitCast(
2838 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2839 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2840 /*IsInitializer=*/true);
2841 ArgVal = CtorCGF.EmitLoadOfScalar(
2842 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2843 CGM.getContext().VoidPtrTy, Dst.getLocation());
2844 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2845 CtorCGF.FinishFunction();
2846 Ctor = Fn;
2847 }
2848 if (VD->getType().isDestructedType() != QualType::DK_none) {
2849 // Generate function that emits destructor call for the threadprivate copy
2850 // of the variable VD
2851 CodeGenFunction DtorCGF(CGM);
2852 FunctionArgList Args;
2853 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2854 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2855 ImplicitParamDecl::Other);
2856 Args.push_back(&Dst);
2857
2858 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2859 CGM.getContext().VoidTy, Args);
2860 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2861 std::string Name = getName({"__kmpc_global_dtor_", ""});
2862 llvm::Function *Fn =
2863 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2864 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2865 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2866 Loc, Loc);
2867 // Create a scope with an artificial location for the body of this function.
2868 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2869 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2870 DtorCGF.GetAddrOfLocalVar(&Dst),
2871 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2872 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2873 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2874 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2875 DtorCGF.FinishFunction();
2876 Dtor = Fn;
2877 }
2878 // Do not emit init function if it is not required.
2879 if (!Ctor && !Dtor)
2880 return nullptr;
2881
2882 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2883 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2884 /*isVarArg=*/false)
2885 ->getPointerTo();
2886 // Copying constructor for the threadprivate variable.
2887 // Must be NULL - reserved by runtime, but currently it requires that this
2888 // parameter is always NULL. Otherwise it fires assertion.
2889 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2890 if (Ctor == nullptr) {
2891 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2892 /*isVarArg=*/false)
2893 ->getPointerTo();
2894 Ctor = llvm::Constant::getNullValue(CtorTy);
2895 }
2896 if (Dtor == nullptr) {
2897 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2898 /*isVarArg=*/false)
2899 ->getPointerTo();
2900 Dtor = llvm::Constant::getNullValue(DtorTy);
2901 }
2902 if (!CGF) {
2903 auto *InitFunctionTy =
2904 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2905 std::string Name = getName({"__omp_threadprivate_init_", ""});
2906 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2907 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2908 CodeGenFunction InitCGF(CGM);
2909 FunctionArgList ArgList;
2910 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2911 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2912 Loc, Loc);
2913 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2914 InitCGF.FinishFunction();
2915 return InitFunction;
2916 }
2917 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2918 }
2919 return nullptr;
2920}
2921
2922bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2923 llvm::GlobalVariable *Addr,
2924 bool PerformInit) {
2925 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2926 !CGM.getLangOpts().OpenMPIsDevice)
2927 return false;
2928 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2929 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2930 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2931 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2932 HasRequiresUnifiedSharedMemory))
2933 return CGM.getLangOpts().OpenMPIsDevice;
2934 VD = VD->getDefinition(CGM.getContext());
2935 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2936 return CGM.getLangOpts().OpenMPIsDevice;
2937
2938 QualType ASTTy = VD->getType();
2939
2940 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2941 // Produce the unique prefix to identify the new target regions. We use
2942 // the source location of the variable declaration which we know to not
2943 // conflict with any target region.
2944 unsigned DeviceID;
2945 unsigned FileID;
2946 unsigned Line;
2947 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2948 SmallString<128> Buffer, Out;
2949 {
2950 llvm::raw_svector_ostream OS(Buffer);
2951 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2952 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2953 }
2954
2955 const Expr *Init = VD->getAnyInitializer();
2956 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2957 llvm::Constant *Ctor;
2958 llvm::Constant *ID;
2959 if (CGM.getLangOpts().OpenMPIsDevice) {
2960 // Generate function that re-emits the declaration's initializer into
2961 // the threadprivate copy of the variable VD
2962 CodeGenFunction CtorCGF(CGM);
2963
2964 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2965 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2966 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2967 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2968 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2969 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2970 FunctionArgList(), Loc, Loc);
2971 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2972 CtorCGF.EmitAnyExprToMem(Init,
2973 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2974 Init->getType().getQualifiers(),
2975 /*IsInitializer=*/true);
2976 CtorCGF.FinishFunction();
2977 Ctor = Fn;
2978 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2979 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2980 } else {
2981 Ctor = new llvm::GlobalVariable(
2982 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2983 llvm::GlobalValue::PrivateLinkage,
2984 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2985 ID = Ctor;
2986 }
2987
2988 // Register the information for the entry associated with the constructor.
2989 Out.clear();
2990 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2991 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2992 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2993 }
2994 if (VD->getType().isDestructedType() != QualType::DK_none) {
2995 llvm::Constant *Dtor;
2996 llvm::Constant *ID;
2997 if (CGM.getLangOpts().OpenMPIsDevice) {
2998 // Generate function that emits destructor call for the threadprivate
2999 // copy of the variable VD
3000 CodeGenFunction DtorCGF(CGM);
3001
3002 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3003 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3004 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
3005 FTy, Twine(Buffer, "_dtor"), FI, Loc);
3006 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
3007 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
3008 FunctionArgList(), Loc, Loc);
3009 // Create a scope with an artificial location for the body of this
3010 // function.
3011 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
3012 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
3013 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
3014 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
3015 DtorCGF.FinishFunction();
3016 Dtor = Fn;
3017 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
3018 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
3019 } else {
3020 Dtor = new llvm::GlobalVariable(
3021 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
3022 llvm::GlobalValue::PrivateLinkage,
3023 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
3024 ID = Dtor;
3025 }
3026 // Register the information for the entry associated with the destructor.
3027 Out.clear();
3028 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
3029 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
3030 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
3031 }
3032 return CGM.getLangOpts().OpenMPIsDevice;
3033}
3034
3035Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
3036 QualType VarType,
3037 StringRef Name) {
3038 std::string Suffix = getName({"artificial", ""});
3039 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
3040 llvm::Value *GAddr =
3041 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
3042 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
3043 CGM.getTarget().isTLSSupported()) {
3044 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
3045 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
3046 }
3047 std::string CacheSuffix = getName({"cache", ""});
3048 llvm::Value *Args[] = {
3049 emitUpdateLocation(CGF, SourceLocation()),
3050 getThreadID(CGF, SourceLocation()),
3051 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
3052 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
3053 /*isSigned=*/false),
3054 getOrCreateInternalVariable(
3055 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
3056 return Address(
3057 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3058 CGF.EmitRuntimeCall(
3059 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3060 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3061 CGM.getContext().getTypeAlignInChars(VarType));
3062}
3063
3064void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
3065 const RegionCodeGenTy &ThenGen,
3066 const RegionCodeGenTy &ElseGen) {
3067 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3068
3069 // If the condition constant folds and can be elided, try to avoid emitting
3070 // the condition and the dead arm of the if/else.
3071 bool CondConstant;
3072 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
3073 if (CondConstant)
3074 ThenGen(CGF);
3075 else
3076 ElseGen(CGF);
3077 return;
3078 }
3079
3080 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3081 // emit the conditional branch.
3082 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3083 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3084 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3085 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3086
3087 // Emit the 'then' code.
3088 CGF.EmitBlock(ThenBlock);
3089 ThenGen(CGF);
3090 CGF.EmitBranch(ContBlock);
3091 // Emit the 'else' code if present.
3092 // There is no need to emit line number for unconditional branch.
3093 (void)ApplyDebugLocation::CreateEmpty(CGF);
3094 CGF.EmitBlock(ElseBlock);
3095 ElseGen(CGF);
3096 // There is no need to emit line number for unconditional branch.
3097 (void)ApplyDebugLocation::CreateEmpty(CGF);
3098 CGF.EmitBranch(ContBlock);
3099 // Emit the continuation block for code after the if.
3100 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3101}
3102
3103void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3104 llvm::Function *OutlinedFn,
3105 ArrayRef<llvm::Value *> CapturedVars,
3106 const Expr *IfCond) {
3107 if (!CGF.HaveInsertPoint())
3108 return;
3109 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3110 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3111 PrePostActionTy &) {
3112 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3113 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3114 llvm::Value *Args[] = {
3115 RTLoc,
3116 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3117 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3118 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3119 RealArgs.append(std::begin(Args), std::end(Args));
3120 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3121
3122 llvm::FunctionCallee RTLFn =
3123 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3124 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3125 };
3126 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3127 PrePostActionTy &) {
3128 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3129 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3130 // Build calls:
3131 // __kmpc_serialized_parallel(&Loc, GTid);
3132 llvm::Value *Args[] = {RTLoc, ThreadID};
3133 CGF.EmitRuntimeCall(
3134 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3135
3136 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
3137 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
3138 Address ZeroAddrBound =
3139 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3140 /*Name=*/".bound.zero.addr");
3141 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
3142 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3143 // ThreadId for serialized parallels is 0.
3144 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
3145 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
3146 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3147 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3148
3149 // __kmpc_end_serialized_parallel(&Loc, GTid);
3150 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3151 CGF.EmitRuntimeCall(
3152 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3153 EndArgs);
3154 };
3155 if (IfCond) {
3156 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3157 } else {
3158 RegionCodeGenTy ThenRCG(ThenGen);
3159 ThenRCG(CGF);
3160 }
3161}
3162
3163// If we're inside an (outlined) parallel region, use the region info's
3164// thread-ID variable (it is passed in a first argument of the outlined function
3165// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3166// regular serial code region, get thread ID by calling kmp_int32
3167// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3168// return the address of that temp.
3169Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3170 SourceLocation Loc) {
3171 if (auto *OMPRegionInfo =
3172 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3173 if (OMPRegionInfo->getThreadIDVariable())
3174 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3175
3176 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3177 QualType Int32Ty =
3178 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3179 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3180 CGF.EmitStoreOfScalar(ThreadID,
3181 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3182
3183 return ThreadIDTemp;
3184}
3185
3186llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3187 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3188 SmallString<256> Buffer;
3189 llvm::raw_svector_ostream Out(Buffer);
3190 Out << Name;
3191 StringRef RuntimeName = Out.str();
3192 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3193 if (Elem.second) {
3194 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3195, __PRETTY_FUNCTION__))
3195 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3195, __PRETTY_FUNCTION__))
;
3196 return &*Elem.second;
3197 }
3198
3199 return Elem.second = new llvm::GlobalVariable(
3200 CGM.getModule(), Ty, /*IsConstant*/ false,
3201 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3202 Elem.first(), /*InsertBefore=*/nullptr,
3203 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3204}
3205
3206llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3207 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3208 std::string Name = getName({Prefix, "var"});
3209 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3210}
3211
3212namespace {
3213/// Common pre(post)-action for different OpenMP constructs.
3214class CommonActionTy final : public PrePostActionTy {
3215 llvm::FunctionCallee EnterCallee;
3216 ArrayRef<llvm::Value *> EnterArgs;
3217 llvm::FunctionCallee ExitCallee;
3218 ArrayRef<llvm::Value *> ExitArgs;
3219 bool Conditional;
3220 llvm::BasicBlock *ContBlock = nullptr;
3221
3222public:
3223 CommonActionTy(llvm::FunctionCallee EnterCallee,
3224 ArrayRef<llvm::Value *> EnterArgs,
3225 llvm::FunctionCallee ExitCallee,
3226 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3227 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3228 ExitArgs(ExitArgs), Conditional(Conditional) {}
3229 void Enter(CodeGenFunction &CGF) override {
3230 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3231 if (Conditional) {
3232 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3233 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3234 ContBlock = CGF.createBasicBlock("omp_if.end");
3235 // Generate the branch (If-stmt)
3236 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3237 CGF.EmitBlock(ThenBlock);
3238 }
3239 }
3240 void Done(CodeGenFunction &CGF) {
3241 // Emit the rest of blocks/branches
3242 CGF.EmitBranch(ContBlock);
3243 CGF.EmitBlock(ContBlock, true);
3244 }
3245 void Exit(CodeGenFunction &CGF) override {
3246 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3247 }
3248};
3249} // anonymous namespace
3250
3251void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3252 StringRef CriticalName,
3253 const RegionCodeGenTy &CriticalOpGen,
3254 SourceLocation Loc, const Expr *Hint) {
3255 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3256 // CriticalOpGen();
3257 // __kmpc_end_critical(ident_t *, gtid, Lock);
3258 // Prepare arguments and build a call to __kmpc_critical
3259 if (!CGF.HaveInsertPoint())
3260 return;
3261 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3262 getCriticalRegionLock(CriticalName)};
3263 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3264 std::end(Args));
3265 if (Hint) {
3266 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3267 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3268 }
3269 CommonActionTy Action(
3270 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3271 : OMPRTL__kmpc_critical),
3272 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3273 CriticalOpGen.setAction(Action);
3274 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3275}
3276
3277void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3278 const RegionCodeGenTy &MasterOpGen,
3279 SourceLocation Loc) {
3280 if (!CGF.HaveInsertPoint())
3281 return;
3282 // if(__kmpc_master(ident_t *, gtid)) {
3283 // MasterOpGen();
3284 // __kmpc_end_master(ident_t *, gtid);
3285 // }
3286 // Prepare arguments and build a call to __kmpc_master
3287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3288 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3289 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3290 /*Conditional=*/true);
3291 MasterOpGen.setAction(Action);
3292 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3293 Action.Done(CGF);
3294}
3295
3296void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3297 SourceLocation Loc) {
3298 if (!CGF.HaveInsertPoint())
3299 return;
3300 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3301 if (OMPBuilder) {
3302 OMPBuilder->CreateTaskyield(CGF.Builder);
3303 } else {
3304 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3305 llvm::Value *Args[] = {
3306 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3307 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3308 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield),
3309 Args);
3310 }
3311
3312 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3313 Region->emitUntiedSwitch(CGF);
3314}
3315
3316void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3317 const RegionCodeGenTy &TaskgroupOpGen,
3318 SourceLocation Loc) {
3319 if (!CGF.HaveInsertPoint())
3320 return;
3321 // __kmpc_taskgroup(ident_t *, gtid);
3322 // TaskgroupOpGen();
3323 // __kmpc_end_taskgroup(ident_t *, gtid);
3324 // Prepare arguments and build a call to __kmpc_taskgroup
3325 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3326 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3327 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3328 Args);
3329 TaskgroupOpGen.setAction(Action);
3330 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3331}
3332
3333/// Given an array of pointers to variables, project the address of a
3334/// given variable.
3335static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3336 unsigned Index, const VarDecl *Var) {
3337 // Pull out the pointer to the variable.
3338 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3339 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3340
3341 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3342 Addr = CGF.Builder.CreateElementBitCast(
3343 Addr, CGF.ConvertTypeForMem(Var->getType()));
3344 return Addr;
3345}
3346
3347static llvm::Value *emitCopyprivateCopyFunction(
3348 CodeGenModule &CGM, llvm::Type *ArgsType,
3349 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3350 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3351 SourceLocation Loc) {
3352 ASTContext &C = CGM.getContext();
3353 // void copy_func(void *LHSArg, void *RHSArg);
3354 FunctionArgList Args;
3355 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3356 ImplicitParamDecl::Other);
3357 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3358 ImplicitParamDecl::Other);
3359 Args.push_back(&LHSArg);
3360 Args.push_back(&RHSArg);
3361 const auto &CGFI =
3362 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3363 std::string Name =
3364 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3365 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3366 llvm::GlobalValue::InternalLinkage, Name,
3367 &CGM.getModule());
3368 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3369 Fn->setDoesNotRecurse();
3370 CodeGenFunction CGF(CGM);
3371 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3372 // Dest = (void*[n])(LHSArg);
3373 // Src = (void*[n])(RHSArg);
3374 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3375 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3376 ArgsType), CGF.getPointerAlign());
3377 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3378 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3379 ArgsType), CGF.getPointerAlign());
3380 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3381 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3382 // ...
3383 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3384 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3385 const auto *DestVar =
3386 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3387 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3388
3389 const auto *SrcVar =
3390 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3391 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3392
3393 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3394 QualType Type = VD->getType();
3395 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3396 }
3397 CGF.FinishFunction();
3398 return Fn;
3399}
3400
3401void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3402 const RegionCodeGenTy &SingleOpGen,
3403 SourceLocation Loc,
3404 ArrayRef<const Expr *> CopyprivateVars,
3405 ArrayRef<const Expr *> SrcExprs,
3406 ArrayRef<const Expr *> DstExprs,
3407 ArrayRef<const Expr *> AssignmentOps) {
3408 if (!CGF.HaveInsertPoint())
3409 return;
3410 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3412, __PRETTY_FUNCTION__))
3411 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3412, __PRETTY_FUNCTION__))
3412 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3412, __PRETTY_FUNCTION__))
;
3413 ASTContext &C = CGM.getContext();
3414 // int32 did_it = 0;
3415 // if(__kmpc_single(ident_t *, gtid)) {
3416 // SingleOpGen();
3417 // __kmpc_end_single(ident_t *, gtid);
3418 // did_it = 1;
3419 // }
3420 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3421 // <copy_func>, did_it);
3422
3423 Address DidIt = Address::invalid();
3424 if (!CopyprivateVars.empty()) {
3425 // int32 did_it = 0;
3426 QualType KmpInt32Ty =
3427 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3428 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3429 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3430 }
3431 // Prepare arguments and build a call to __kmpc_single
3432 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3433 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3434 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3435 /*Conditional=*/true);
3436 SingleOpGen.setAction(Action);
3437 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3438 if (DidIt.isValid()) {
3439 // did_it = 1;
3440 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3441 }
3442 Action.Done(CGF);
3443 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3444 // <copy_func>, did_it);
3445 if (DidIt.isValid()) {
3446 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3447 QualType CopyprivateArrayTy = C.getConstantArrayType(
3448 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3449 /*IndexTypeQuals=*/0);
3450 // Create a list of all private variables for copyprivate.
3451 Address CopyprivateList =
3452 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3453 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3454 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3455 CGF.Builder.CreateStore(
3456 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3457 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
3458 CGF.VoidPtrTy),
3459 Elem);
3460 }
3461 // Build function that copies private values from single region to all other
3462 // threads in the corresponding parallel region.
3463 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3464 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3465 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3466 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3467 Address CL =
3468 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3469 CGF.VoidPtrTy);
3470 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3471 llvm::Value *Args[] = {
3472 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3473 getThreadID(CGF, Loc), // i32 <gtid>
3474 BufSize, // size_t <buf_size>
3475 CL.getPointer(), // void *<copyprivate list>
3476 CpyFn, // void (*) (void *, void *) <copy_func>
3477 DidItVal // i32 did_it
3478 };
3479 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3480 }
3481}
3482
3483void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3484 const RegionCodeGenTy &OrderedOpGen,
3485 SourceLocation Loc, bool IsThreads) {
3486 if (!CGF.HaveInsertPoint())
3487 return;
3488 // __kmpc_ordered(ident_t *, gtid);
3489 // OrderedOpGen();
3490 // __kmpc_end_ordered(ident_t *, gtid);
3491 // Prepare arguments and build a call to __kmpc_ordered
3492 if (IsThreads) {
3493 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3494 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3495 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3496 Args);
3497 OrderedOpGen.setAction(Action);
3498 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3499 return;
3500 }
3501 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3502}
3503
3504unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3505 unsigned Flags;
3506 if (Kind == OMPD_for)
3507 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3508 else if (Kind == OMPD_sections)
3509 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3510 else if (Kind == OMPD_single)
3511 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3512 else if (Kind == OMPD_barrier)
3513 Flags = OMP_IDENT_BARRIER_EXPL;
3514 else
3515 Flags = OMP_IDENT_BARRIER_IMPL;
3516 return Flags;
3517}
3518
3519void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3520 CodeGenFunction &CGF, const OMPLoopDirective &S,
3521 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3522 // Check if the loop directive is actually a doacross loop directive. In this
3523 // case choose static, 1 schedule.
3524 if (llvm::any_of(
3525 S.getClausesOfKind<OMPOrderedClause>(),
3526 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3527 ScheduleKind = OMPC_SCHEDULE_static;
3528 // Chunk size is 1 in this case.
3529 llvm::APInt ChunkSize(32, 1);
3530 ChunkExpr = IntegerLiteral::Create(
3531 CGF.getContext(), ChunkSize,
3532 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3533 SourceLocation());
3534 }
3535}
3536
3537void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3538 OpenMPDirectiveKind Kind, bool EmitChecks,
3539 bool ForceSimpleCall) {
3540 // Check if we should use the OMPBuilder
3541 auto *OMPRegionInfo =
3542 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
3543 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3544 if (OMPBuilder) {
3545 CGF.Builder.restoreIP(OMPBuilder->CreateBarrier(
3546 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
3547 return;
3548 }
3549
3550 if (!CGF.HaveInsertPoint())
3551 return;
3552 // Build call __kmpc_cancel_barrier(loc, thread_id);
3553 // Build call __kmpc_barrier(loc, thread_id);
3554 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3555 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3556 // thread_id);
3557 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3558 getThreadID(CGF, Loc)};
3559 if (OMPRegionInfo) {
3560 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3561 llvm::Value *Result = CGF.EmitRuntimeCall(
3562 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3563 if (EmitChecks) {
3564 // if (__kmpc_cancel_barrier()) {
3565 // exit from construct;
3566 // }
3567 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3568 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3569 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3570 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3571 CGF.EmitBlock(ExitBB);
3572 // exit from construct;
3573 CodeGenFunction::JumpDest CancelDestination =
3574 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3575 CGF.EmitBranchThroughCleanup(CancelDestination);
3576 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3577 }
3578 return;
3579 }
3580 }
3581 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3582}
3583
3584/// Map the OpenMP loop schedule to the runtime enumeration.
3585static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3586 bool Chunked, bool Ordered) {
3587 switch (ScheduleKind) {
3588 case OMPC_SCHEDULE_static:
3589 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3590 : (Ordered ? OMP_ord_static : OMP_sch_static);
3591 case OMPC_SCHEDULE_dynamic:
3592 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3593 case OMPC_SCHEDULE_guided:
3594 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3595 case OMPC_SCHEDULE_runtime:
3596 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3597 case OMPC_SCHEDULE_auto:
3598 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3599 case OMPC_SCHEDULE_unknown:
3600 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3600, __PRETTY_FUNCTION__))
;
3601 return Ordered ? OMP_ord_static : OMP_sch_static;
3602 }
3603 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3603)
;
3604}
3605
3606/// Map the OpenMP distribute schedule to the runtime enumeration.
3607static OpenMPSchedType
3608getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3609 // only static is allowed for dist_schedule
3610 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3611}
3612
3613bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3614 bool Chunked) const {
3615 OpenMPSchedType Schedule =
3616 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3617 return Schedule == OMP_sch_static;
3618}
3619
3620bool CGOpenMPRuntime::isStaticNonchunked(
3621 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3622 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3623 return Schedule == OMP_dist_sch_static;
3624}
3625
3626bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3627 bool Chunked) const {
3628 OpenMPSchedType Schedule =
3629 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3630 return Schedule == OMP_sch_static_chunked;
3631}
3632
3633bool CGOpenMPRuntime::isStaticChunked(
3634 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3635 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3636 return Schedule == OMP_dist_sch_static_chunked;
3637}
3638
3639bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3640 OpenMPSchedType Schedule =
3641 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3642 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3642, __PRETTY_FUNCTION__))
;
3643 return Schedule != OMP_sch_static;
3644}
3645
3646static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3647 OpenMPScheduleClauseModifier M1,
3648 OpenMPScheduleClauseModifier M2) {
3649 int Modifier = 0;
3650 switch (M1) {
3651 case OMPC_SCHEDULE_MODIFIER_monotonic:
3652 Modifier = OMP_sch_modifier_monotonic;
3653 break;
3654 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3655 Modifier = OMP_sch_modifier_nonmonotonic;
3656 break;
3657 case OMPC_SCHEDULE_MODIFIER_simd:
3658 if (Schedule == OMP_sch_static_chunked)
3659 Schedule = OMP_sch_static_balanced_chunked;
3660 break;
3661 case OMPC_SCHEDULE_MODIFIER_last:
3662 case OMPC_SCHEDULE_MODIFIER_unknown:
3663 break;
3664 }
3665 switch (M2) {
3666 case OMPC_SCHEDULE_MODIFIER_monotonic:
3667 Modifier = OMP_sch_modifier_monotonic;
3668 break;
3669 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3670 Modifier = OMP_sch_modifier_nonmonotonic;
3671 break;
3672 case OMPC_SCHEDULE_MODIFIER_simd:
3673 if (Schedule == OMP_sch_static_chunked)
3674 Schedule = OMP_sch_static_balanced_chunked;
3675 break;
3676 case OMPC_SCHEDULE_MODIFIER_last:
3677 case OMPC_SCHEDULE_MODIFIER_unknown:
3678 break;
3679 }
3680 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3681 // If the static schedule kind is specified or if the ordered clause is
3682 // specified, and if the nonmonotonic modifier is not specified, the effect is
3683 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3684 // modifier is specified, the effect is as if the nonmonotonic modifier is
3685 // specified.
3686 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3687 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3688 Schedule == OMP_sch_static_balanced_chunked ||
3689 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
3690 Schedule == OMP_dist_sch_static_chunked ||
3691 Schedule == OMP_dist_sch_static))
3692 Modifier = OMP_sch_modifier_nonmonotonic;
3693 }
3694 return Schedule | Modifier;
3695}
3696
3697void CGOpenMPRuntime::emitForDispatchInit(
3698 CodeGenFunction &CGF, SourceLocation Loc,
3699 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3700 bool Ordered, const DispatchRTInput &DispatchValues) {
3701 if (!CGF.HaveInsertPoint())
3702 return;
3703 OpenMPSchedType Schedule = getRuntimeSchedule(
3704 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3705 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
3706 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
3707 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
3708 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3708, __PRETTY_FUNCTION__))
;
3709 // Call __kmpc_dispatch_init(
3710 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3711 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3712 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3713
3714 // If the Chunk was not specified in the clause - use default value 1.
3715 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3716 : CGF.Builder.getIntN(IVSize, 1);
3717 llvm::Value *Args[] = {
3718 emitUpdateLocation(CGF, Loc),
3719 getThreadID(CGF, Loc),
3720 CGF.Builder.getInt32(addMonoNonMonoModifier(
3721 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3722 DispatchValues.LB, // Lower
3723 DispatchValues.UB, // Upper
3724 CGF.Builder.getIntN(IVSize, 1), // Stride
3725 Chunk // Chunk
3726 };
3727 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3728}
3729
3730static void emitForStaticInitCall(
3731 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3732 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3733 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3734 const CGOpenMPRuntime::StaticRTInput &Values) {
3735 if (!CGF.HaveInsertPoint())
3736 return;
3737
3738 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3738, __PRETTY_FUNCTION__))
;
3739 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3743, __PRETTY_FUNCTION__))
3740 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3743, __PRETTY_FUNCTION__))
3741 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3743, __PRETTY_FUNCTION__))
3742 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3743, __PRETTY_FUNCTION__))
3743 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3743, __PRETTY_FUNCTION__))
;
3744
3745 // Call __kmpc_for_static_init(
3746 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3747 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3748 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3749 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3750 llvm::Value *Chunk = Values.Chunk;
3751 if (Chunk == nullptr) {
3752 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3754, __PRETTY_FUNCTION__))
3753 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3754, __PRETTY_FUNCTION__))
3754 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3754, __PRETTY_FUNCTION__))
;
3755 // If the Chunk was not specified in the clause - use default value 1.
3756 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3757 } else {
3758 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3762, __PRETTY_FUNCTION__))
3759 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3762, __PRETTY_FUNCTION__))
3760 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3762, __PRETTY_FUNCTION__))
3761 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3762, __PRETTY_FUNCTION__))
3762 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3762, __PRETTY_FUNCTION__))
;
3763 }
3764 llvm::Value *Args[] = {
3765 UpdateLocation,
3766 ThreadId,
3767 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3768 M2)), // Schedule type
3769 Values.IL.getPointer(), // &isLastIter
3770 Values.LB.getPointer(), // &LB
3771 Values.UB.getPointer(), // &UB
3772 Values.ST.getPointer(), // &Stride
3773 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3774 Chunk // Chunk
3775 };
3776 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3777}
3778
3779void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3780 SourceLocation Loc,
3781 OpenMPDirectiveKind DKind,
3782 const OpenMPScheduleTy &ScheduleKind,
3783 const StaticRTInput &Values) {
3784 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3785 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3786 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3787, __PRETTY_FUNCTION__))
3787 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3787, __PRETTY_FUNCTION__))
;
3788 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3789 isOpenMPLoopDirective(DKind)
3790 ? OMP_IDENT_WORK_LOOP
3791 : OMP_IDENT_WORK_SECTIONS);
3792 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3793 llvm::FunctionCallee StaticInitFunction =
3794 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3795 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3796 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3797 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3798}
3799
3800void CGOpenMPRuntime::emitDistributeStaticInit(
3801 CodeGenFunction &CGF, SourceLocation Loc,
3802 OpenMPDistScheduleClauseKind SchedKind,
3803 const CGOpenMPRuntime::StaticRTInput &Values) {
3804 OpenMPSchedType ScheduleNum =
3805 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3806 llvm::Value *UpdatedLocation =
3807 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3808 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3809 llvm::FunctionCallee StaticInitFunction =
3810 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3811 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3812 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3813 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3814}
3815
3816void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3817 SourceLocation Loc,
3818 OpenMPDirectiveKind DKind) {
3819 if (!CGF.HaveInsertPoint())
3820 return;
3821 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3822 llvm::Value *Args[] = {
3823 emitUpdateLocation(CGF, Loc,
3824 isOpenMPDistributeDirective(DKind)
3825 ? OMP_IDENT_WORK_DISTRIBUTE
3826 : isOpenMPLoopDirective(DKind)
3827 ? OMP_IDENT_WORK_LOOP
3828 : OMP_IDENT_WORK_SECTIONS),
3829 getThreadID(CGF, Loc)};
3830 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3831 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3832 Args);
3833}
3834
3835void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3836 SourceLocation Loc,
3837 unsigned IVSize,
3838 bool IVSigned) {
3839 if (!CGF.HaveInsertPoint())
3840 return;
3841 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3842 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3843 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3844}
3845
3846llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3847 SourceLocation Loc, unsigned IVSize,
3848 bool IVSigned, Address IL,
3849 Address LB, Address UB,
3850 Address ST) {
3851 // Call __kmpc_dispatch_next(
3852 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3853 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3854 // kmp_int[32|64] *p_stride);
3855 llvm::Value *Args[] = {
3856 emitUpdateLocation(CGF, Loc),
3857 getThreadID(CGF, Loc),
3858 IL.getPointer(), // &isLastIter
3859 LB.getPointer(), // &Lower
3860 UB.getPointer(), // &Upper
3861 ST.getPointer() // &Stride
3862 };
3863 llvm::Value *Call =
3864 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3865 return CGF.EmitScalarConversion(
3866 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3867 CGF.getContext().BoolTy, Loc);
3868}
3869
3870void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3871 llvm::Value *NumThreads,
3872 SourceLocation Loc) {
3873 if (!CGF.HaveInsertPoint())
3874 return;
3875 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3876 llvm::Value *Args[] = {
3877 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3878 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3879 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3880 Args);
3881}
3882
3883void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3884 ProcBindKind ProcBind,
3885 SourceLocation Loc) {
3886 if (!CGF.HaveInsertPoint())
3887 return;
3888 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3888, __PRETTY_FUNCTION__))
;
3889 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3890 llvm::Value *Args[] = {
3891 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3892 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
3893 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3894}
3895
3896void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3897 SourceLocation Loc, llvm::AtomicOrdering AO) {
3898 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3899 if (OMPBuilder) {
3900 OMPBuilder->CreateFlush(CGF.Builder);
3901 } else {
3902 if (!CGF.HaveInsertPoint())
3903 return;
3904 // Build call void __kmpc_flush(ident_t *loc)
3905 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3906 emitUpdateLocation(CGF, Loc));
3907 }
3908}
3909
3910namespace {
3911/// Indexes of fields for type kmp_task_t.
3912enum KmpTaskTFields {
3913 /// List of shared variables.
3914 KmpTaskTShareds,
3915 /// Task routine.
3916 KmpTaskTRoutine,
3917 /// Partition id for the untied tasks.
3918 KmpTaskTPartId,
3919 /// Function with call of destructors for private variables.
3920 Data1,
3921 /// Task priority.
3922 Data2,
3923 /// (Taskloops only) Lower bound.
3924 KmpTaskTLowerBound,
3925 /// (Taskloops only) Upper bound.
3926 KmpTaskTUpperBound,
3927 /// (Taskloops only) Stride.
3928 KmpTaskTStride,
3929 /// (Taskloops only) Is last iteration flag.
3930 KmpTaskTLastIter,
3931 /// (Taskloops only) Reduction data.
3932 KmpTaskTReductions,
3933};
3934} // anonymous namespace
3935
3936bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3937 return OffloadEntriesTargetRegion.empty() &&
3938 OffloadEntriesDeviceGlobalVar.empty();
3939}
3940
3941/// Initialize target region entry.
3942void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3943 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3944 StringRef ParentName, unsigned LineNum,
3945 unsigned Order) {
3946 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3948, __PRETTY_FUNCTION__))
3947 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3948, __PRETTY_FUNCTION__))
3948 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3948, __PRETTY_FUNCTION__))
;
3949 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3950 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3951 OMPTargetRegionEntryTargetRegion);
3952 ++OffloadingEntriesNum;
3953}
3954
3955void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3956 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3957 StringRef ParentName, unsigned LineNum,
3958 llvm::Constant *Addr, llvm::Constant *ID,
3959 OMPTargetRegionEntryKind Flags) {
3960 // If we are emitting code for a target, the entry is already initialized,
3961 // only has to be registered.
3962 if (CGM.getLangOpts().OpenMPIsDevice) {
3963 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3964 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3965 DiagnosticsEngine::Error,
3966 "Unable to find target region on line '%0' in the device code.");
3967 CGM.getDiags().Report(DiagID) << LineNum;
3968 return;
3969 }
3970 auto &Entry =
3971 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3972 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3972, __PRETTY_FUNCTION__))
;
3973 Entry.setAddress(Addr);
3974 Entry.setID(ID);
3975 Entry.setFlags(Flags);
3976 } else {
3977 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3978 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3979 ++OffloadingEntriesNum;
3980 }
3981}
3982
3983bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3984 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3985 unsigned LineNum) const {
3986 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3987 if (PerDevice == OffloadEntriesTargetRegion.end())
3988 return false;
3989 auto PerFile = PerDevice->second.find(FileID);
3990 if (PerFile == PerDevice->second.end())
3991 return false;
3992 auto PerParentName = PerFile->second.find(ParentName);
3993 if (PerParentName == PerFile->second.end())
3994 return false;
3995 auto PerLine = PerParentName->second.find(LineNum);
3996 if (PerLine == PerParentName->second.end())
3997 return false;
3998 // Fail if this entry is already registered.
3999 if (PerLine->second.getAddress() || PerLine->second.getID())
4000 return false;
4001 return true;
4002}
4003
4004void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
4005 const OffloadTargetRegionEntryInfoActTy &Action) {
4006 // Scan all target region entries and perform the provided action.
4007 for (const auto &D : OffloadEntriesTargetRegion)
4008 for (const auto &F : D.second)
4009 for (const auto &P : F.second)
4010 for (const auto &L : P.second)
4011 Action(D.first, F.first, P.first(), L.first, L.second);
4012}
4013
4014void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4015 initializeDeviceGlobalVarEntryInfo(StringRef Name,
4016 OMPTargetGlobalVarEntryKind Flags,
4017 unsigned Order) {
4018 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4020, __PRETTY_FUNCTION__))
4019 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4020, __PRETTY_FUNCTION__))
4020 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4020, __PRETTY_FUNCTION__))
;
4021 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
4022 ++OffloadingEntriesNum;
4023}
4024
4025void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4026 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
4027 CharUnits VarSize,
4028 OMPTargetGlobalVarEntryKind Flags,
4029 llvm::GlobalValue::LinkageTypes Linkage) {
4030 if (CGM.getLangOpts().OpenMPIsDevice) {
4031 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4032 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4033, __PRETTY_FUNCTION__))
4033 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4033, __PRETTY_FUNCTION__))
;
4034 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4035, __PRETTY_FUNCTION__))
4035 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4035, __PRETTY_FUNCTION__))
;
4036 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
4037 if (Entry.getVarSize().isZero()) {
4038 Entry.setVarSize(VarSize);
4039 Entry.setLinkage(Linkage);
4040 }
4041 return;
4042 }
4043 Entry.setVarSize(VarSize);
4044 Entry.setLinkage(Linkage);
4045 Entry.setAddress(Addr);
4046 } else {
4047 if (hasDeviceGlobalVarEntryInfo(VarName)) {
4048 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4049 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4050, __PRETTY_FUNCTION__))
4050 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4050, __PRETTY_FUNCTION__))
;
4051 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4052, __PRETTY_FUNCTION__))
4052 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4052, __PRETTY_FUNCTION__))
;
4053 if (Entry.getVarSize().isZero()) {
4054 Entry.setVarSize(VarSize);
4055 Entry.setLinkage(Linkage);
4056 }
4057 return;
4058 }
4059 OffloadEntriesDeviceGlobalVar.try_emplace(
4060 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4061 ++OffloadingEntriesNum;
4062 }
4063}
4064
4065void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4066 actOnDeviceGlobalVarEntriesInfo(
4067 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4068 // Scan all target region entries and perform the provided action.
4069 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4070 Action(E.getKey(), E.getValue());
4071}
4072
4073void CGOpenMPRuntime::createOffloadEntry(
4074 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4075 llvm::GlobalValue::LinkageTypes Linkage) {
4076 StringRef Name = Addr->getName();
4077 llvm::Module &M = CGM.getModule();
4078 llvm::LLVMContext &C = M.getContext();
4079
4080 // Create constant string with the name.
4081 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4082
4083 std::string StringName = getName({"omp_offloading", "entry_name"});
4084 auto *Str = new llvm::GlobalVariable(
4085 M, StrPtrInit->getType(), /*isConstant=*/true,
4086 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4087 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4088
4089 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4090 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4091 llvm::ConstantInt::get(CGM.SizeTy, Size),
4092 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4093 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4094 std::string EntryName = getName({"omp_offloading", "entry", ""});
4095 llvm::GlobalVariable *Entry = createGlobalStruct(
4096 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4097 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4098
4099 // The entry has to be created in the section the linker expects it to be.
4100 Entry->setSection("omp_offloading_entries");
4101}
4102
4103void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4104 // Emit the offloading entries and metadata so that the device codegen side
4105 // can easily figure out what to emit. The produced metadata looks like
4106 // this:
4107 //
4108 // !omp_offload.info = !{!1, ...}
4109 //
4110 // Right now we only generate metadata for function that contain target
4111 // regions.
4112
4113 // If we are in simd mode or there are no entries, we don't need to do
4114 // anything.
4115 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
4116 return;
4117
4118 llvm::Module &M = CGM.getModule();
4119 llvm::LLVMContext &C = M.getContext();
4120 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4121 SourceLocation, StringRef>,
4122 16>
4123 OrderedEntries(OffloadEntriesInfoManager.size());
4124 llvm::SmallVector<StringRef, 16> ParentFunctions(
4125 OffloadEntriesInfoManager.size());
4126
4127 // Auxiliary methods to create metadata values and strings.
4128 auto &&GetMDInt = [this](unsigned V) {
4129 return llvm::ConstantAsMetadata::get(
4130 llvm::ConstantInt::get(CGM.Int32Ty, V));
4131 };
4132
4133 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4134
4135 // Create the offloading info metadata node.
4136 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4137
4138 // Create function that emits metadata for each target region entry;
4139 auto &&TargetRegionMetadataEmitter =
4140 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4141 &GetMDString](
4142 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4143 unsigned Line,
4144 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4145 // Generate metadata for target regions. Each entry of this metadata
4146 // contains:
4147 // - Entry 0 -> Kind of this type of metadata (0).
4148 // - Entry 1 -> Device ID of the file where the entry was identified.
4149 // - Entry 2 -> File ID of the file where the entry was identified.
4150 // - Entry 3 -> Mangled name of the function where the entry was
4151 // identified.
4152 // - Entry 4 -> Line in the file where the entry was identified.
4153 // - Entry 5 -> Order the entry was created.
4154 // The first element of the metadata node is the kind.
4155 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4156 GetMDInt(FileID), GetMDString(ParentName),
4157 GetMDInt(Line), GetMDInt(E.getOrder())};
4158
4159 SourceLocation Loc;
4160 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4161 E = CGM.getContext().getSourceManager().fileinfo_end();
4162 I != E; ++I) {
4163 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4164 I->getFirst()->getUniqueID().getFile() == FileID) {
4165 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4166 I->getFirst(), Line, 1);
4167 break;
4168 }
4169 }
4170 // Save this entry in the right position of the ordered entries array.
4171 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4172 ParentFunctions[E.getOrder()] = ParentName;
4173
4174 // Add metadata to the named metadata node.
4175 MD->addOperand(llvm::MDNode::get(C, Ops));
4176 };
4177
4178 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4179 TargetRegionMetadataEmitter);
4180
4181 // Create function that emits metadata for each device global variable entry;
4182 auto &&DeviceGlobalVarMetadataEmitter =
4183 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4184 MD](StringRef MangledName,
4185 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4186 &E) {
4187 // Generate metadata for global variables. Each entry of this metadata
4188 // contains:
4189 // - Entry 0 -> Kind of this type of metadata (1).
4190 // - Entry 1 -> Mangled name of the variable.
4191 // - Entry 2 -> Declare target kind.
4192 // - Entry 3 -> Order the entry was created.
4193 // The first element of the metadata node is the kind.
4194 llvm::Metadata *Ops[] = {
4195 GetMDInt(E.getKind()), GetMDString(MangledName),
4196 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4197
4198 // Save this entry in the right position of the ordered entries array.
4199 OrderedEntries[E.getOrder()] =
4200 std::make_tuple(&E, SourceLocation(), MangledName);
4201
4202 // Add metadata to the named metadata node.
4203 MD->addOperand(llvm::MDNode::get(C, Ops));
4204 };
4205
4206 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4207 DeviceGlobalVarMetadataEmitter);
4208
4209 for (const auto &E : OrderedEntries) {
4210 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4210, __PRETTY_FUNCTION__))
;
4211 if (const auto *CE =
4212 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4213 std::get<0>(E))) {
4214 if (!CE->getID() || !CE->getAddress()) {
4215 // Do not blame the entry if the parent funtion is not emitted.
4216 StringRef FnName = ParentFunctions[CE->getOrder()];
4217 if (!CGM.GetGlobalValue(FnName))
4218 continue;
4219 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4220 DiagnosticsEngine::Error,
4221 "Offloading entry for target region in %0 is incorrect: either the "
4222 "address or the ID is invalid.");
4223 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4224 continue;
4225 }
4226 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4227 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4228 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
4229 OffloadEntryInfoDeviceGlobalVar>(
4230 std::get<0>(E))) {
4231 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4232 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4233 CE->getFlags());
4234 switch (Flags) {
4235 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4236 if (CGM.getLangOpts().OpenMPIsDevice &&
4237 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4238 continue;
4239 if (!CE->getAddress()) {
4240 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4241 DiagnosticsEngine::Error, "Offloading entry for declare target "
4242 "variable %0 is incorrect: the "
4243 "address is invalid.");
4244 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4245 continue;
4246 }
4247 // The vaiable has no definition - no need to add the entry.
4248 if (CE->getVarSize().isZero())
4249 continue;
4250 break;
4251 }
4252 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4253 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4255, __PRETTY_FUNCTION__))
4254 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4255, __PRETTY_FUNCTION__))
4255 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4255, __PRETTY_FUNCTION__))
;
4256 if (CGM.getLangOpts().OpenMPIsDevice)
4257 continue;
4258 if (!CE->getAddress()) {
4259 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4260 DiagnosticsEngine::Error,
4261 "Offloading entry for declare target variable is incorrect: the "
4262 "address is invalid.");
4263 CGM.getDiags().Report(DiagID);
4264 continue;
4265 }
4266 break;
4267 }
4268 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4269 CE->getVarSize().getQuantity(), Flags,
4270 CE->getLinkage());
4271 } else {
4272 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4272)
;
4273 }
4274 }
4275}
4276
4277/// Loads all the offload entries information from the host IR
4278/// metadata.
4279void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4280 // If we are in target mode, load the metadata from the host IR. This code has
4281 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4282
4283 if (!CGM.getLangOpts().OpenMPIsDevice)
4284 return;
4285
4286 if (CGM.getLangOpts().OMPHostIRFile.empty())
4287 return;
4288
4289 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4290 if (auto EC = Buf.getError()) {
4291 CGM.getDiags().Report(diag::err_cannot_open_file)
4292 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4293 return;
4294 }
4295
4296 llvm::LLVMContext C;
4297 auto ME = expectedToErrorOrAndEmitErrors(
4298 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4299
4300 if (auto EC = ME.getError()) {
4301 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4302 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4303 CGM.getDiags().Report(DiagID)
4304 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4305 return;
4306 }
4307
4308 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4309 if (!MD)
4310 return;
4311
4312 for (llvm::MDNode *MN : MD->operands()) {
4313 auto &&GetMDInt = [MN](unsigned Idx) {
4314 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4315 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4316 };
4317
4318 auto &&GetMDString = [MN](unsigned Idx) {
4319 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4320 return V->getString();
4321 };
4322
4323 switch (GetMDInt(0)) {
4324 default:
4325 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4325)
;
4326 break;
4327 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4328 OffloadingEntryInfoTargetRegion:
4329 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4330 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4331 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4332 /*Order=*/GetMDInt(5));
4333 break;
4334 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4335 OffloadingEntryInfoDeviceGlobalVar:
4336 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4337 /*MangledName=*/GetMDString(1),
4338 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4339 /*Flags=*/GetMDInt(2)),
4340 /*Order=*/GetMDInt(3));
4341 break;
4342 }
4343 }
4344}
4345
4346void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4347 if (!KmpRoutineEntryPtrTy) {
4348 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4349 ASTContext &C = CGM.getContext();
4350 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4351 FunctionProtoType::ExtProtoInfo EPI;
4352 KmpRoutineEntryPtrQTy = C.getPointerType(
4353 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4354 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4355 }
4356}
4357
4358QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4359 // Make sure the type of the entry is already created. This is the type we
4360 // have to create:
4361 // struct __tgt_offload_entry{
4362 // void *addr; // Pointer to the offload entry info.
4363 // // (function or global)
4364 // char *name; // Name of the function or global.
4365 // size_t size; // Size of the entry info (0 if it a function).
4366 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4367 // int32_t reserved; // Reserved, to use by the runtime library.
4368 // };
4369 if (TgtOffloadEntryQTy.isNull()) {
4370 ASTContext &C = CGM.getContext();
4371 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4372 RD->startDefinition();
4373 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4374 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4375 addFieldToRecordDecl(C, RD, C.getSizeType());
4376 addFieldToRecordDecl(
4377 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4378 addFieldToRecordDecl(
4379 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4380 RD->completeDefinition();
4381 RD->addAttr(PackedAttr::CreateImplicit(C));
4382 TgtOffloadEntryQTy = C.getRecordType(RD);
4383 }
4384 return TgtOffloadEntryQTy;
4385}
4386
4387namespace {
4388struct PrivateHelpersTy {
4389 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4390 const VarDecl *PrivateElemInit)
4391 : Original(Original), PrivateCopy(PrivateCopy),
4392 PrivateElemInit(PrivateElemInit) {}
4393 const VarDecl *Original;
4394 const VarDecl *PrivateCopy;
4395 const VarDecl *PrivateElemInit;
4396};
4397typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4398} // anonymous namespace
4399
4400static RecordDecl *
4401createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4402 if (!Privates.empty()) {
4403 ASTContext &C = CGM.getContext();
4404 // Build struct .kmp_privates_t. {
4405 // /* private vars */
4406 // };
4407 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4408 RD->startDefinition();
4409 for (const auto &Pair : Privates) {
4410 const VarDecl *VD = Pair.second.Original;
4411 QualType Type = VD->getType().getNonReferenceType();
4412 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4413 if (VD->hasAttrs()) {
4414 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4415 E(VD->getAttrs().end());
4416 I != E; ++I)
4417 FD->addAttr(*I);
4418 }
4419 }
4420 RD->completeDefinition();
4421 return RD;
4422 }
4423 return nullptr;
4424}
4425
4426static RecordDecl *
4427createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4428 QualType KmpInt32Ty,
4429 QualType KmpRoutineEntryPointerQTy) {
4430 ASTContext &C = CGM.getContext();
4431 // Build struct kmp_task_t {
4432 // void * shareds;
4433 // kmp_routine_entry_t routine;
4434 // kmp_int32 part_id;
4435 // kmp_cmplrdata_t data1;
4436 // kmp_cmplrdata_t data2;
4437 // For taskloops additional fields:
4438 // kmp_uint64 lb;
4439 // kmp_uint64 ub;
4440 // kmp_int64 st;
4441 // kmp_int32 liter;
4442 // void * reductions;
4443 // };
4444 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4445 UD->startDefinition();
4446 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4447 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4448 UD->completeDefinition();
4449 QualType KmpCmplrdataTy = C.getRecordType(UD);
4450 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4451 RD->startDefinition();
4452 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4453 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4454 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4455 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4456 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4457 if (isOpenMPTaskLoopDirective(Kind)) {
4458 QualType KmpUInt64Ty =
4459 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4460 QualType KmpInt64Ty =
4461 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4462 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4463 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4464 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4465 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4466 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4467 }
4468 RD->completeDefinition();
4469 return RD;
4470}
4471
4472static RecordDecl *
4473createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4474 ArrayRef<PrivateDataTy> Privates) {
4475 ASTContext &C = CGM.getContext();
4476 // Build struct kmp_task_t_with_privates {
4477 // kmp_task_t task_data;
4478 // .kmp_privates_t. privates;
4479 // };
4480 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4481 RD->startDefinition();
4482 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4483 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4484 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4485 RD->completeDefinition();
4486 return RD;
4487}
4488
4489/// Emit a proxy function which accepts kmp_task_t as the second
4490/// argument.
4491/// \code
4492/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4493/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4494/// For taskloops:
4495/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4496/// tt->reductions, tt->shareds);
4497/// return 0;
4498/// }
4499/// \endcode
4500static llvm::Function *
4501emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4502 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4503 QualType KmpTaskTWithPrivatesPtrQTy,
4504 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4505 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4506 llvm::Value *TaskPrivatesMap) {
4507 ASTContext &C = CGM.getContext();
4508 FunctionArgList Args;
4509 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4510 ImplicitParamDecl::Other);
4511 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4512 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4513 ImplicitParamDecl::Other);
4514 Args.push_back(&GtidArg);
4515 Args.push_back(&TaskTypeArg);
4516 const auto &TaskEntryFnInfo =
4517 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4518 llvm::FunctionType *TaskEntryTy =
4519 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4520 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4521 auto *TaskEntry = llvm::Function::Create(
4522 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4523 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4524 TaskEntry->setDoesNotRecurse();
4525 CodeGenFunction CGF(CGM);
4526 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4527 Loc, Loc);
4528
4529 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4530 // tt,
4531 // For taskloops:
4532 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4533 // tt->task_data.shareds);
4534 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4535 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4536 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4537 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4538 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4539 const auto *KmpTaskTWithPrivatesQTyRD =
4540 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4541 LValue Base =
4542 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4543 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4544 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4545 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4546 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4547
4548 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4549 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4550 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4551 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4552 CGF.ConvertTypeForMem(SharedsPtrTy));
4553
4554 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4555 llvm::Value *PrivatesParam;
4556 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4557 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4558 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4559 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
4560 } else {
4561 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4562 }
4563
4564 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4565 TaskPrivatesMap,
4566 CGF.Builder
4567 .CreatePointerBitCastOrAddrSpaceCast(
4568 TDBase.getAddress(CGF), CGF.VoidPtrTy)
4569 .getPointer()};
4570 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4571 std::end(CommonArgs));
4572 if (isOpenMPTaskLoopDirective(Kind)) {
4573 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4574 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4575 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4576 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4577 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4578 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4579 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4580 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4581 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4582 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4583 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4584 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4585 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4586 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4587 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4588 CallArgs.push_back(LBParam);
4589 CallArgs.push_back(UBParam);
4590 CallArgs.push_back(StParam);
4591 CallArgs.push_back(LIParam);
4592 CallArgs.push_back(RParam);
4593 }
4594 CallArgs.push_back(SharedsParam);
4595
4596 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4597 CallArgs);
4598 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4599 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4600 CGF.FinishFunction();
4601 return TaskEntry;
4602}
4603
4604static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4605 SourceLocation Loc,
4606 QualType KmpInt32Ty,
4607 QualType KmpTaskTWithPrivatesPtrQTy,
4608 QualType KmpTaskTWithPrivatesQTy) {
4609 ASTContext &C = CGM.getContext();
4610 FunctionArgList Args;
4611 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4612 ImplicitParamDecl::Other);
4613 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4614 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4615 ImplicitParamDecl::Other);
4616 Args.push_back(&GtidArg);
4617 Args.push_back(&TaskTypeArg);
4618 const auto &DestructorFnInfo =
4619 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4620 llvm::FunctionType *DestructorFnTy =
4621 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4622 std::string Name =
4623 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4624 auto *DestructorFn =
4625 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4626 Name, &CGM.getModule());
4627 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4628 DestructorFnInfo);
4629 DestructorFn->setDoesNotRecurse();
4630 CodeGenFunction CGF(CGM);
4631 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4632 Args, Loc, Loc);
4633
4634 LValue Base = CGF.EmitLoadOfPointerLValue(
4635 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4636 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4637 const auto *KmpTaskTWithPrivatesQTyRD =
4638 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4639 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4640 Base = CGF.EmitLValueForField(Base, *FI);
4641 for (const auto *Field :
4642 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4643 if (QualType::DestructionKind DtorKind =
4644 Field->getType().isDestructedType()) {
4645 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4646 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
4647 }
4648 }
4649 CGF.FinishFunction();
4650 return DestructorFn;
4651}
4652
4653/// Emit a privates mapping function for correct handling of private and
4654/// firstprivate variables.
4655/// \code
4656/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4657/// **noalias priv1,..., <tyn> **noalias privn) {
4658/// *priv1 = &.privates.priv1;
4659/// ...;
4660/// *privn = &.privates.privn;
4661/// }
4662/// \endcode
4663static llvm::Value *
4664emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4665 ArrayRef<const Expr *> PrivateVars,
4666 ArrayRef<const Expr *> FirstprivateVars,
4667 ArrayRef<const Expr *> LastprivateVars,
4668 QualType PrivatesQTy,
4669 ArrayRef<PrivateDataTy> Privates) {
4670 ASTContext &C = CGM.getContext();
4671 FunctionArgList Args;
4672 ImplicitParamDecl TaskPrivatesArg(
4673 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4674 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4675 ImplicitParamDecl::Other);
4676 Args.push_back(&TaskPrivatesArg);
4677 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4678 unsigned Counter = 1;
4679 for (const Expr *E : PrivateVars) {
4680 Args.push_back(ImplicitParamDecl::Create(
4681 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4682 C.getPointerType(C.getPointerType(E->getType()))
4683 .withConst()
4684 .withRestrict(),
4685 ImplicitParamDecl::Other));
4686 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4687 PrivateVarsPos[VD] = Counter;
4688 ++Counter;
4689 }
4690 for (const Expr *E : FirstprivateVars) {
4691 Args.push_back(ImplicitParamDecl::Create(
4692 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4693 C.getPointerType(C.getPointerType(E->getType()))
4694 .withConst()
4695 .withRestrict(),
4696 ImplicitParamDecl::Other));
4697 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4698 PrivateVarsPos[VD] = Counter;
4699 ++Counter;
4700 }
4701 for (const Expr *E : LastprivateVars) {
4702 Args.push_back(ImplicitParamDecl::Create(
4703 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4704 C.getPointerType(C.getPointerType(E->getType()))
4705 .withConst()
4706 .withRestrict(),
4707 ImplicitParamDecl::Other));
4708 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4709 PrivateVarsPos[VD] = Counter;
4710 ++Counter;
4711 }
4712 const auto &TaskPrivatesMapFnInfo =
4713 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4714 llvm::FunctionType *TaskPrivatesMapTy =
4715 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4716 std::string Name =
4717 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4718 auto *TaskPrivatesMap = llvm::Function::Create(
4719 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4720 &CGM.getModule());
4721 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4722 TaskPrivatesMapFnInfo);
4723 if (CGM.getLangOpts().Optimize) {
4724 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4725 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4726 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4727 }
4728 CodeGenFunction CGF(CGM);
4729 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4730 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4731
4732 // *privi = &.privates.privi;
4733 LValue Base = CGF.EmitLoadOfPointerLValue(
4734 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4735 TaskPrivatesArg.getType()->castAs<PointerType>());
4736 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4737 Counter = 0;
4738 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4739 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4740 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4741 LValue RefLVal =
4742 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4743 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4744 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
4745 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
4746 ++Counter;
4747 }
4748 CGF.FinishFunction();
4749 return TaskPrivatesMap;
4750}
4751
4752/// Emit initialization for private variables in task-based directives.
4753static void emitPrivatesInit(CodeGenFunction &CGF,
4754 const OMPExecutableDirective &D,
4755 Address KmpTaskSharedsPtr, LValue TDBase,
4756 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4757 QualType SharedsTy, QualType SharedsPtrTy,
4758 const OMPTaskDataTy &Data,
4759 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4760 ASTContext &C = CGF.getContext();
4761 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4762 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4763 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4764 ? OMPD_taskloop
4765 : OMPD_task;
4766 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4767 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4768 LValue SrcBase;
4769 bool IsTargetTask =
4770 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4771 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4772 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4773 // PointersArray and SizesArray. The original variables for these arrays are
4774 // not captured and we get their addresses explicitly.
4775 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4776 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4777 SrcBase = CGF.MakeAddrLValue(
4778 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4779 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4780 SharedsTy);
4781 }
4782 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4783 for (const PrivateDataTy &Pair : Privates) {
4784 const VarDecl *VD = Pair.second.PrivateCopy;
4785 const Expr *Init = VD->getAnyInitializer();
4786 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4787 !CGF.isTrivialInitializer(Init)))) {
4788 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4789 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4790 const VarDecl *OriginalVD = Pair.second.Original;
4791 // Check if the variable is the target-based BasePointersArray,
4792 // PointersArray or SizesArray.
4793 LValue SharedRefLValue;
4794 QualType Type = PrivateLValue.getType();
4795 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4796 if (IsTargetTask && !SharedField) {
4797 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4798 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4799 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4800 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4801 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4802 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4803 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
4804 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4804, __PRETTY_FUNCTION__))
;
4805 SharedRefLValue =
4806 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4807 } else {
4808 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4809 SharedRefLValue = CGF.MakeAddrLValue(
4810 Address(SharedRefLValue.getPointer(CGF),
4811 C.getDeclAlign(OriginalVD)),
4812 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4813 SharedRefLValue.getTBAAInfo());
4814 }
4815 if (Type->isArrayType()) {
4816 // Initialize firstprivate array.
4817 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4818 // Perform simple memcpy.
4819 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4820 } else {
4821 // Initialize firstprivate array using element-by-element
4822 // initialization.
4823 CGF.EmitOMPAggregateAssign(
4824 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
4825 Type,
4826 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4827 Address SrcElement) {
4828 // Clean up any temporaries needed by the initialization.
4829 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4830 InitScope.addPrivate(
4831 Elem, [SrcElement]() -> Address { return SrcElement; });
4832 (void)InitScope.Privatize();
4833 // Emit initialization for single element.
4834 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4835 CGF, &CapturesInfo);
4836 CGF.EmitAnyExprToMem(Init, DestElement,
4837 Init->getType().getQualifiers(),
4838 /*IsInitializer=*/false);
4839 });
4840 }
4841 } else {
4842 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4843 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
4844 return SharedRefLValue.getAddress(CGF);
4845 });
4846 (void)InitScope.Privatize();
4847 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4848 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4849 /*capturedByInit=*/false);
4850 }
4851 } else {
4852 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4853 }
4854 }
4855 ++FI;
4856 }
4857}
4858
4859/// Check if duplication function is required for taskloops.
4860static bool checkInitIsRequired(CodeGenFunction &CGF,
4861 ArrayRef<PrivateDataTy> Privates) {
4862 bool InitRequired = false;
4863 for (const PrivateDataTy &Pair : Privates) {
4864 const VarDecl *VD = Pair.second.PrivateCopy;
4865 const Expr *Init = VD->getAnyInitializer();
4866 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4867 !CGF.isTrivialInitializer(Init));
4868 if (InitRequired)
4869 break;
4870 }
4871 return InitRequired;
4872}
4873
4874
4875/// Emit task_dup function (for initialization of
4876/// private/firstprivate/lastprivate vars and last_iter flag)
4877/// \code
4878/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4879/// lastpriv) {
4880/// // setup lastprivate flag
4881/// task_dst->last = lastpriv;
4882/// // could be constructor calls here...
4883/// }
4884/// \endcode
4885static llvm::Value *
4886emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4887 const OMPExecutableDirective &D,
4888 QualType KmpTaskTWithPrivatesPtrQTy,
4889 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4890 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4891 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4892 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4893 ASTContext &C = CGM.getContext();
4894 FunctionArgList Args;
4895 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4896 KmpTaskTWithPrivatesPtrQTy,
4897 ImplicitParamDecl::Other);
4898 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4899 KmpTaskTWithPrivatesPtrQTy,
4900 ImplicitParamDecl::Other);
4901 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4902 ImplicitParamDecl::Other);
4903 Args.push_back(&DstArg);
4904 Args.push_back(&SrcArg);
4905 Args.push_back(&LastprivArg);
4906 const auto &TaskDupFnInfo =
4907 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4908 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4909 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4910 auto *TaskDup = llvm::Function::Create(
4911 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4912 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4913 TaskDup->setDoesNotRecurse();
4914 CodeGenFunction CGF(CGM);
4915 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4916 Loc);
4917
4918 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4919 CGF.GetAddrOfLocalVar(&DstArg),
4920 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4921 // task_dst->liter = lastpriv;
4922 if (WithLastIter) {
4923 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4924 LValue Base = CGF.EmitLValueForField(
4925 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4926 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4927 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4928 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4929 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4930 }
4931
4932 // Emit initial values for private copies (if any).
4933 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4933, __PRETTY_FUNCTION__))
;
4934 Address KmpTaskSharedsPtr = Address::invalid();
4935 if (!Data.FirstprivateVars.empty()) {
4936 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4937 CGF.GetAddrOfLocalVar(&SrcArg),
4938 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4939 LValue Base = CGF.EmitLValueForField(
4940 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4941 KmpTaskSharedsPtr = Address(
4942 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4943 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4944 KmpTaskTShareds)),
4945 Loc),
4946 CGF.getNaturalTypeAlignment(SharedsTy));
4947 }
4948 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4949 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4950 CGF.FinishFunction();
4951 return TaskDup;
4952}
4953
4954/// Checks if destructor function is required to be generated.
4955/// \return true if cleanups are required, false otherwise.
4956static bool
4957checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4958 bool NeedsCleanup = false;
4959 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4960 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4961 for (const FieldDecl *FD : PrivateRD->fields()) {
4962 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4963 if (NeedsCleanup)
4964 break;
4965 }
4966 return NeedsCleanup;
4967}
4968
4969CGOpenMPRuntime::TaskResultTy
4970CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4971 const OMPExecutableDirective &D,
4972 llvm::Function *TaskFunction, QualType SharedsTy,
4973 Address Shareds, const OMPTaskDataTy &Data) {
4974 ASTContext &C = CGM.getContext();
4975 llvm::SmallVector<PrivateDataTy, 4> Privates;
4976 // Aggregate privates and sort them by the alignment.
4977 auto I = Data.PrivateCopies.begin();
4978 for (const Expr *E : Data.PrivateVars) {
4979 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4980 Privates.emplace_back(
4981 C.getDeclAlign(VD),
4982 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4983 /*PrivateElemInit=*/nullptr));
4984 ++I;
4985 }
4986 I = Data.FirstprivateCopies.begin();
4987 auto IElemInitRef = Data.FirstprivateInits.begin();
4988 for (const Expr *E : Data.FirstprivateVars) {
4989 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4990 Privates.emplace_back(
4991 C.getDeclAlign(VD),
4992 PrivateHelpersTy(
4993 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4994 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4995 ++I;
4996 ++IElemInitRef;
4997 }
4998 I = Data.LastprivateCopies.begin();
4999 for (const Expr *E : Data.LastprivateVars) {
5000 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5001 Privates.emplace_back(
5002 C.getDeclAlign(VD),
5003 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5004 /*PrivateElemInit=*/nullptr));
5005 ++I;
5006 }
5007 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5008 return L.first > R.first;
5009 });
5010 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5011 // Build type kmp_routine_entry_t (if not built yet).
5012 emitKmpRoutineEntryT(KmpInt32Ty);
5013 // Build type kmp_task_t (if not built yet).
5014 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5015 if (SavedKmpTaskloopTQTy.isNull()) {
5016 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5017 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5018 }
5019 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5020 } else {
5021 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5024, __PRETTY_FUNCTION__))
5022 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5024, __PRETTY_FUNCTION__))
5023 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5024, __PRETTY_FUNCTION__))
5024 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5024, __PRETTY_FUNCTION__))
;
5025 if (SavedKmpTaskTQTy.isNull()) {
5026 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5027 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5028 }
5029 KmpTaskTQTy = SavedKmpTaskTQTy;
5030 }
5031 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5032 // Build particular struct kmp_task_t for the given task.
5033 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5034 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5035 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5036 QualType KmpTaskTWithPrivatesPtrQTy =
5037 C.getPointerType(KmpTaskTWithPrivatesQTy);
5038 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5039 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5040 KmpTaskTWithPrivatesTy->getPointerTo();
5041 llvm::Value *KmpTaskTWithPrivatesTySize =
5042 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5043 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5044
5045 // Emit initial values for private copies (if any).
5046 llvm::Value *TaskPrivatesMap = nullptr;
5047 llvm::Type *TaskPrivatesMapTy =
5048 std::next(TaskFunction->arg_begin(), 3)->getType();
5049 if (!Privates.empty()) {
5050 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5051 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5052 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5053 FI->getType(), Privates);
5054 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5055 TaskPrivatesMap, TaskPrivatesMapTy);
5056 } else {
5057 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5058 cast<llvm::PointerType>(TaskPrivatesMapTy));
5059 }
5060 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5061 // kmp_task_t *tt);
5062 llvm::Function *TaskEntry = emitProxyTaskFunction(
5063 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5064 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5065 TaskPrivatesMap);
5066
5067 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5068 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5069 // kmp_routine_entry_t *task_entry);
5070 // Task flags. Format is taken from
5071 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5072 // description of kmp_tasking_flags struct.
5073 enum {
5074 TiedFlag = 0x1,
5075 FinalFlag = 0x2,
5076 DestructorsFlag = 0x8,
5077 PriorityFlag = 0x20
5078 };
5079 unsigned Flags = Data.Tied ? TiedFlag : 0;
5080 bool NeedsCleanup = false;
5081 if (!Privates.empty()) {
5082 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5083 if (NeedsCleanup)
5084 Flags = Flags | DestructorsFlag;
5085 }
5086 if (Data.Priority.getInt())
5087 Flags = Flags | PriorityFlag;
5088 llvm::Value *TaskFlags =
5089 Data.Final.getPointer()
5090 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5091 CGF.Builder.getInt32(FinalFlag),
5092 CGF.Builder.getInt32(/*C=*/0))
5093 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5094 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5095 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5096 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5097 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5098 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5099 TaskEntry, KmpRoutineEntryPtrTy)};
5100 llvm::Value *NewTask;
5101 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5102 // Check if we have any device clause associated with the directive.
5103 const Expr *Device = nullptr;
5104 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5105 Device = C->getDevice();
5106 // Emit device ID if any otherwise use default value.
5107 llvm::Value *DeviceID;
5108 if (Device)
5109 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5110 CGF.Int64Ty, /*isSigned=*/true);
5111 else
5112 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5113 AllocArgs.push_back(DeviceID);
5114 NewTask = CGF.EmitRuntimeCall(
5115 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5116 } else {
5117 NewTask = CGF.EmitRuntimeCall(
5118 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5119 }
5120 llvm::Value *NewTaskNewTaskTTy =
5121 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5122 NewTask, KmpTaskTWithPrivatesPtrTy);
5123 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5124 KmpTaskTWithPrivatesQTy);
5125 LValue TDBase =
5126 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5127 // Fill the data in the resulting kmp_task_t record.
5128 // Copy shareds if there are any.
5129 Address KmpTaskSharedsPtr = Address::invalid();
5130 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5131 KmpTaskSharedsPtr =
5132 Address(CGF.EmitLoadOfScalar(
5133 CGF.EmitLValueForField(
5134 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5135 KmpTaskTShareds)),
5136 Loc),
5137 CGF.getNaturalTypeAlignment(SharedsTy));
5138 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5139 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5140 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5141 }
5142 // Emit initial values for private copies (if any).
5143 TaskResultTy Result;
5144 if (!Privates.empty()) {
5145 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5146 SharedsTy, SharedsPtrTy, Data, Privates,
5147 /*ForDup=*/false);
5148 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5149 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5150 Result.TaskDupFn = emitTaskDupFunction(
5151 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5152 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5153 /*WithLastIter=*/!Data.LastprivateVars.empty());
5154 }
5155 }
5156 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5157 enum { Priority = 0, Destructors = 1 };
5158 // Provide pointer to function with destructors for privates.
5159 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5160 const RecordDecl *KmpCmplrdataUD =
5161 (*FI)->getType()->getAsUnionType()->getDecl();
5162 if (NeedsCleanup) {
5163 llvm::Value *DestructorFn = emitDestructorsFunction(
5164 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5165 KmpTaskTWithPrivatesQTy);
5166 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5167 LValue DestructorsLV = CGF.EmitLValueForField(
5168 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5169 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5170 DestructorFn, KmpRoutineEntryPtrTy),
5171 DestructorsLV);
5172 }
5173 // Set priority.
5174 if (Data.Priority.getInt()) {
5175 LValue Data2LV = CGF.EmitLValueForField(
5176 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5177 LValue PriorityLV = CGF.EmitLValueForField(
5178 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5179 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5180 }
5181 Result.NewTask = NewTask;
5182 Result.TaskEntry = TaskEntry;
5183 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5184 Result.TDBase = TDBase;
5185 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5186 return Result;
5187}
5188
5189namespace {
5190/// Dependence kind for RTL.
5191enum RTLDependenceKindTy {
5192 DepIn = 0x01,
5193 DepInOut = 0x3,
5194 DepMutexInOutSet = 0x4
5195};
5196/// Fields ids in kmp_depend_info record.
5197enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5198} // namespace
5199
5200/// Translates internal dependency kind into the runtime kind.
5201static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
5202 RTLDependenceKindTy DepKind;
5203 switch (K) {
5204 case OMPC_DEPEND_in:
5205 DepKind = DepIn;
5206 break;
5207 // Out and InOut dependencies must use the same code.
5208 case OMPC_DEPEND_out:
5209 case OMPC_DEPEND_inout:
5210 DepKind = DepInOut;
5211 break;
5212 case OMPC_DEPEND_mutexinoutset:
5213 DepKind = DepMutexInOutSet;
5214 break;
5215 case OMPC_DEPEND_source:
5216 case OMPC_DEPEND_sink:
5217 case OMPC_DEPEND_depobj:
5218 case OMPC_DEPEND_unknown:
5219 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5219)
;
5220 }
5221 return DepKind;
5222}
5223
5224/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
5225static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
5226 QualType &FlagsTy) {
5227 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5228 if (KmpDependInfoTy.isNull()) {
5229 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5230 KmpDependInfoRD->startDefinition();
5231 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5232 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5233 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5234 KmpDependInfoRD->completeDefinition();
5235 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5236 }
5237}
5238
5239Address CGOpenMPRuntime::emitDependClause(
5240 CodeGenFunction &CGF,
5241 ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependencies,
5242 bool ForDepobj, SourceLocation Loc) {
5243 // Process list of dependencies.
5244 ASTContext &C = CGM.getContext();
5245 Address DependenciesArray = Address::invalid();
5246 unsigned NumDependencies = Dependencies.size();
5247 if (NumDependencies) {
5248 QualType FlagsTy;
5249 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5250 RecordDecl *KmpDependInfoRD =
5251 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5252 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5253 // Define type kmp_depend_info[<Dependencies.size()>];
5254 // For depobj reserve one extra element to store the number of elements.
5255 // It is required to handle depobj(x) update(in) construct.
5256 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5257 KmpDependInfoTy,
5258 llvm::APInt(/*numBits=*/64, NumDependencies + (ForDepobj ? 1 : 0)),
5259 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
5260 // kmp_depend_info[<Dependencies.size()>] deps;
5261 if (ForDepobj) {
5262 // Need to allocate on the dynamic memory.
5263 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5264 // Use default allocator.
5265 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5266 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoArrayTy);
5267 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
5268 llvm::Value *Size = CGF.CGM.getSize(Sz.alignTo(Align));
5269 llvm::Value *Args[] = {ThreadID, Size, Allocator};
5270
5271 llvm::Value *Addr = CGF.EmitRuntimeCall(
5272 createRuntimeFunction(OMPRTL__kmpc_alloc), Args, ".dep.arr.addr");
5273 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5274 Addr, CGF.ConvertTypeForMem(KmpDependInfoArrayTy)->getPointerTo());
5275 DependenciesArray = Address(Addr, Align);
5276 } else {
5277 DependenciesArray =
5278 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5279 }
5280 if (ForDepobj) {
5281 // Write number of elements in the first element of array for depobj.
5282 llvm::Value *NumVal =
5283 llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
5284 LValue Base = CGF.MakeAddrLValue(
5285 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0),
5286 KmpDependInfoTy);
5287 // deps[i].base_addr = NumDependencies;
5288 LValue BaseAddrLVal = CGF.EmitLValueForField(
5289 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5290 CGF.EmitStoreOfScalar(NumVal, BaseAddrLVal);
5291 }
5292 for (unsigned I = 0; I < NumDependencies; ++I) {
5293 const Expr *E = Dependencies[I].second;
5294 LValue Addr = CGF.EmitLValue(E);
5295 llvm::Value *Size;
5296 QualType Ty = E->getType();
5297 if (const auto *ASE =
5298 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5299 LValue UpAddrLVal =
5300 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5301 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
5302 UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
5303 llvm::Value *LowIntPtr =
5304 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy);
5305 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5306 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5307 } else {
5308 Size = CGF.getTypeSize(Ty);
5309 }
5310 LValue Base =
5311 CGF.MakeAddrLValue(CGF.Builder.CreateConstArrayGEP(
5312 DependenciesArray, I + (ForDepobj ? 1 : 0)),
5313 KmpDependInfoTy);
5314 // deps[i].base_addr = &<Dependencies[i].second>;
5315 LValue BaseAddrLVal = CGF.EmitLValueForField(
5316 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5317 CGF.EmitStoreOfScalar(
5318 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy),
5319 BaseAddrLVal);
5320 // deps[i].len = sizeof(<Dependencies[i].second>);
5321 LValue LenLVal = CGF.EmitLValueForField(
5322 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5323 CGF.EmitStoreOfScalar(Size, LenLVal);
5324 // deps[i].flags = <Dependencies[i].first>;
5325 RTLDependenceKindTy DepKind =
5326 translateDependencyKind(Dependencies[I].first);
5327 LValue FlagsLVal = CGF.EmitLValueForField(
5328 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5329 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5330 FlagsLVal);
5331 }
5332 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5333 CGF.Builder.CreateConstArrayGEP(DependenciesArray, ForDepobj ? 1 : 0),
5334 CGF.VoidPtrTy);
5335 }
5336 return DependenciesArray;
5337}
5338
5339void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
5340 SourceLocation Loc) {
5341 ASTContext &C = CGM.getContext();
5342 QualType FlagsTy;
5343 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5344 LValue Base = CGF.EmitLoadOfPointerLValue(
5345 DepobjLVal.getAddress(CGF),
5346 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5347 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
5348 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5349 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
5350 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
5351 Addr.getPointer(),
5352 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
5353 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
5354 CGF.VoidPtrTy);
5355 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5356 // Use default allocator.
5357 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5358 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
5359
5360 // _kmpc_free(gtid, addr, nullptr);
5361 (void)CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_free), Args);
5362}
5363
5364void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
5365 OpenMPDependClauseKind NewDepKind,
5366 SourceLocation Loc) {
5367 ASTContext &C = CGM.getContext();
5368 QualType FlagsTy;
5369 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5370 RecordDecl *KmpDependInfoRD =
5371 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5372 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5373 LValue Base = CGF.EmitLoadOfPointerLValue(
5374 DepobjLVal.getAddress(CGF),
5375 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5376 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
5377 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5378 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
5379 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
5380 Base.getTBAAInfo());
5381 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
5382 Addr.getPointer(),
5383 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
5384 LValue NumDepsBase = CGF.MakeAddrLValue(
5385 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
5386 Base.getBaseInfo(), Base.getTBAAInfo());
5387 // NumDeps = deps[i].base_addr;
5388 LValue BaseAddrLVal = CGF.EmitLValueForField(
5389 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5390 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
5391
5392 Address Begin = Base.getAddress(CGF);
5393 // Cast from pointer to array type to pointer to single element.
5394 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps);
5395 // The basic structure here is a while-do loop.
5396 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
5397 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
5398 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5399 CGF.EmitBlock(BodyBB);
5400 llvm::PHINode *ElementPHI =
5401 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
5402 ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
5403 Begin = Address(ElementPHI, Begin.getAlignment());
5404 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
5405 Base.getTBAAInfo());
5406 // deps[i].flags = NewDepKind;
5407 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
5408 LValue FlagsLVal = CGF.EmitLValueForField(
5409 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5410 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5411 FlagsLVal);
5412
5413 // Shift the address forward by one element.
5414 Address ElementNext =
5415 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
5416 ElementPHI->addIncoming(ElementNext.getPointer(),
5417 CGF.Builder.GetInsertBlock());
5418 llvm::Value *IsEmpty =
5419 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
5420 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5421 // Done.
5422 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5423}
5424
5425void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5426 const OMPExecutableDirective &D,
5427 llvm::Function *TaskFunction,
5428 QualType SharedsTy, Address Shareds,
5429 const Expr *IfCond,
5430 const OMPTaskDataTy &Data) {
5431 if (!CGF.HaveInsertPoint())
5432 return;
5433
5434 TaskResultTy Result =
5435 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5436 llvm::Value *NewTask = Result.NewTask;
5437 llvm::Function *TaskEntry = Result.TaskEntry;
5438 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5439 LValue TDBase = Result.TDBase;
5440 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5441 // Process list of dependences.
5442 Address DependenciesArray =
5443 emitDependClause(CGF, Data.Dependences, /*ForDepobj=*/false, Loc);
5444 unsigned NumDependencies = Data.Dependences.size();
5445
5446 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5447 // libcall.
5448 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5449 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5450 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5451 // list is not empty
5452 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5453 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5454 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5455 llvm::Value *DepTaskArgs[7];
5456 if (NumDependencies) {
5457 DepTaskArgs[0] = UpLoc;
5458 DepTaskArgs[1] = ThreadID;
5459 DepTaskArgs[2] = NewTask;
5460 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5461 DepTaskArgs[4] = DependenciesArray.getPointer();
5462 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5463 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5464 }
5465 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5466 &TaskArgs,
5467 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5468 if (!Data.Tied) {
5469 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5470 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5471 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5472 }
5473 if (NumDependencies) {
5474 CGF.EmitRuntimeCall(
5475 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5476 } else {
5477 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5478 TaskArgs);
5479 }
5480 // Check if parent region is untied and build return for untied task;
5481 if (auto *Region =
5482 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5483 Region->emitUntiedSwitch(CGF);
5484 };
5485
5486 llvm::Value *DepWaitTaskArgs[6];
5487 if (NumDependencies) {
5488 DepWaitTaskArgs[0] = UpLoc;
5489 DepWaitTaskArgs[1] = ThreadID;
5490 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5491 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5492 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5493 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5494 }
5495 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5496 NumDependencies, &DepWaitTaskArgs,
5497 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5498 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5499 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5500 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5501 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5502 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5503 // is specified.
5504 if (NumDependencies)
5505 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5506 DepWaitTaskArgs);
5507 // Call proxy_task_entry(gtid, new_task);
5508 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5509 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5510 Action.Enter(CGF);
5511 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5512 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5513 OutlinedFnArgs);
5514 };
5515
5516 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5517 // kmp_task_t *new_task);
5518 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5519 // kmp_task_t *new_task);
5520 RegionCodeGenTy RCG(CodeGen);
5521 CommonActionTy Action(
5522 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5523 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5524 RCG.setAction(Action);
5525 RCG(CGF);
5526 };
5527
5528 if (IfCond) {
5529 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5530 } else {
5531 RegionCodeGenTy ThenRCG(ThenCodeGen);
5532 ThenRCG(CGF);
5533 }
5534}
5535
5536void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5537 const OMPLoopDirective &D,
5538 llvm::Function *TaskFunction,
5539 QualType SharedsTy, Address Shareds,
5540 const Expr *IfCond,
5541 const OMPTaskDataTy &Data) {
5542 if (!CGF.HaveInsertPoint())
5543 return;
5544 TaskResultTy Result =
5545 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5546 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5547 // libcall.
5548 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5549 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5550 // sched, kmp_uint64 grainsize, void *task_dup);
5551 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5552 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5553 llvm::Value *IfVal;
5554 if (IfCond) {
5555 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5556 /*isSigned=*/true);
5557 } else {
5558 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5559 }
5560
5561 LValue LBLVal = CGF.EmitLValueForField(
5562 Result.TDBase,
5563 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5564 const auto *LBVar =
5565 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5566 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5567 LBLVal.getQuals(),
5568 /*IsInitializer=*/true);
5569 LValue UBLVal = CGF.EmitLValueForField(
5570 Result.TDBase,
5571 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5572 const auto *UBVar =
5573 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5574 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5575 UBLVal.getQuals(),
5576 /*IsInitializer=*/true);
5577 LValue StLVal = CGF.EmitLValueForField(
5578 Result.TDBase,
5579 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5580 const auto *StVar =
5581 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5582 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5583 StLVal.getQuals(),
5584 /*IsInitializer=*/true);
5585 // Store reductions address.
5586 LValue RedLVal = CGF.EmitLValueForField(
5587 Result.TDBase,
5588 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5589 if (Data.Reductions) {
5590 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5591 } else {
5592 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5593 CGF.getContext().VoidPtrTy);
5594 }
5595 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5596 llvm::Value *TaskArgs[] = {
5597 UpLoc,
5598 ThreadID,
5599 Result.NewTask,
5600 IfVal,
5601 LBLVal.getPointer(CGF),
5602 UBLVal.getPointer(CGF),
5603 CGF.EmitLoadOfScalar(StLVal, Loc),
5604 llvm::ConstantInt::getSigned(
5605 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5606 llvm::ConstantInt::getSigned(
5607 CGF.IntTy, Data.Schedule.getPointer()
5608 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5609 : NoSchedule),
5610 Data.Schedule.getPointer()
5611 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5612 /*isSigned=*/false)
5613 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5614 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5615 Result.TaskDupFn, CGF.VoidPtrTy)
5616 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5617 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5618}
5619
5620/// Emit reduction operation for each element of array (required for
5621/// array sections) LHS op = RHS.
5622/// \param Type Type of array.
5623/// \param LHSVar Variable on the left side of the reduction operation
5624/// (references element of array in original variable).
5625/// \param RHSVar Variable on the right side of the reduction operation
5626/// (references element of array in original variable).
5627/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5628/// RHSVar.
5629static void EmitOMPAggregateReduction(
5630 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5631 const VarDecl *RHSVar,
5632 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5633 const Expr *, const Expr *)> &RedOpGen,
5634 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5635 const Expr *UpExpr = nullptr) {
5636 // Perform element-by-element initialization.
5637 QualType ElementTy;
5638 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5639 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5640
5641 // Drill down to the base element type on both arrays.
5642 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5643 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5644
5645 llvm::Value *RHSBegin = RHSAddr.getPointer();
5646 llvm::Value *LHSBegin = LHSAddr.getPointer();
5647 // Cast from pointer to array type to pointer to single element.
5648 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5649 // The basic structure here is a while-do loop.
5650 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5651 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5652 llvm::Value *IsEmpty =
5653 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5654 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5655
5656 // Enter the loop body, making that address the current address.
5657 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5658 CGF.EmitBlock(BodyBB);
5659
5660 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5661
5662 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5663 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5664 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5665 Address RHSElementCurrent =
5666 Address(RHSElementPHI,
5667 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5668
5669 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5670 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5671 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5672 Address LHSElementCurrent =
5673 Address(LHSElementPHI,
5674 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5675
5676 // Emit copy.
5677 CodeGenFunction::OMPPrivateScope Scope(CGF);
5678 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5679 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5680 Scope.Privatize();
5681 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5682 Scope.ForceCleanup();
5683
5684 // Shift the address forward by one element.
5685 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5686 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5687 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5688 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5689 // Check whether we've reached the end.
5690 llvm::Value *Done =
5691 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5692 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5693 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5694 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5695
5696 // Done.
5697 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5698}
5699
5700/// Emit reduction combiner. If the combiner is a simple expression emit it as
5701/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5702/// UDR combiner function.
5703static void emitReductionCombiner(CodeGenFunction &CGF,
5704 const Expr *ReductionOp) {
5705 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5706 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5707 if (const auto *DRE =
5708 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5709 if (const auto *DRD =