Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8371, column 15
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fdenormal-fp-math=ieee,ieee -fdenormal-fp-math-f32=ieee,ieee -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/include -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-02-26-193302-13812-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/OpenMPKinds.h"
25#include "clang/CodeGen/ConstantInitBuilder.h"
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/SetOperations.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/DerivedTypes.h"
32#include "llvm/IR/GlobalValue.h"
33#include "llvm/IR/Value.h"
34#include "llvm/Support/AtomicOrdering.h"
35#include "llvm/Support/Format.h"
36#include "llvm/Support/raw_ostream.h"
37#include <cassert>
38
39using namespace clang;
40using namespace CodeGen;
41using namespace llvm::omp;
42
43namespace {
44/// Base class for handling code generation inside OpenMP regions.
45class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
46public:
47 /// Kinds of OpenMP regions used in codegen.
48 enum CGOpenMPRegionKind {
49 /// Region with outlined function for standalone 'parallel'
50 /// directive.
51 ParallelOutlinedRegion,
52 /// Region with outlined function for standalone 'task' directive.
53 TaskOutlinedRegion,
54 /// Region for constructs that do not require function outlining,
55 /// like 'for', 'sections', 'atomic' etc. directives.
56 InlinedRegion,
57 /// Region with outlined function for standalone 'target' directive.
58 TargetRegion,
59 };
60
61 CGOpenMPRegionInfo(const CapturedStmt &CS,
62 const CGOpenMPRegionKind RegionKind,
63 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
64 bool HasCancel)
65 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
66 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
67
68 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
69 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
70 bool HasCancel)
71 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
72 Kind(Kind), HasCancel(HasCancel) {}
73
74 /// Get a variable or parameter for storing global thread id
75 /// inside OpenMP construct.
76 virtual const VarDecl *getThreadIDVariable() const = 0;
77
78 /// Emit the captured statement body.
79 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
80
81 /// Get an LValue for the current ThreadID variable.
82 /// \return LValue for thread id variable. This LValue always has type int32*.
83 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
84
85 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
86
87 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
88
89 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
90
91 bool hasCancel() const { return HasCancel; }
92
93 static bool classof(const CGCapturedStmtInfo *Info) {
94 return Info->getKind() == CR_OpenMP;
95 }
96
97 ~CGOpenMPRegionInfo() override = default;
98
99protected:
100 CGOpenMPRegionKind RegionKind;
101 RegionCodeGenTy CodeGen;
102 OpenMPDirectiveKind Kind;
103 bool HasCancel;
104};
105
106/// API for captured statement code generation in OpenMP constructs.
107class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
108public:
109 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
110 const RegionCodeGenTy &CodeGen,
111 OpenMPDirectiveKind Kind, bool HasCancel,
112 StringRef HelperName)
113 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
114 HasCancel),
115 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
116 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 116, __PRETTY_FUNCTION__))
;
117 }
118
119 /// Get a variable or parameter for storing global thread id
120 /// inside OpenMP construct.
121 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
122
123 /// Get the name of the capture helper.
124 StringRef getHelperName() const override { return HelperName; }
125
126 static bool classof(const CGCapturedStmtInfo *Info) {
127 return CGOpenMPRegionInfo::classof(Info) &&
128 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
129 ParallelOutlinedRegion;
130 }
131
132private:
133 /// A variable or parameter storing global thread id for OpenMP
134 /// constructs.
135 const VarDecl *ThreadIDVar;
136 StringRef HelperName;
137};
138
139/// API for captured statement code generation in OpenMP constructs.
140class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
141public:
142 class UntiedTaskActionTy final : public PrePostActionTy {
143 bool Untied;
144 const VarDecl *PartIDVar;
145 const RegionCodeGenTy UntiedCodeGen;
146 llvm::SwitchInst *UntiedSwitch = nullptr;
147
148 public:
149 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
150 const RegionCodeGenTy &UntiedCodeGen)
151 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
152 void Enter(CodeGenFunction &CGF) override {
153 if (Untied) {
154 // Emit task switching point.
155 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
156 CGF.GetAddrOfLocalVar(PartIDVar),
157 PartIDVar->getType()->castAs<PointerType>());
158 llvm::Value *Res =
159 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
160 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
161 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
162 CGF.EmitBlock(DoneBB);
163 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
164 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
165 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
166 CGF.Builder.GetInsertBlock());
167 emitUntiedSwitch(CGF);
168 }
169 }
170 void emitUntiedSwitch(CodeGenFunction &CGF) const {
171 if (Untied) {
172 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
173 CGF.GetAddrOfLocalVar(PartIDVar),
174 PartIDVar->getType()->castAs<PointerType>());
175 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
176 PartIdLVal);
177 UntiedCodeGen(CGF);
178 CodeGenFunction::JumpDest CurPoint =
179 CGF.getJumpDestInCurrentScope(".untied.next.");
180 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
181 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
182 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
183 CGF.Builder.GetInsertBlock());
184 CGF.EmitBranchThroughCleanup(CurPoint);
185 CGF.EmitBlock(CurPoint.getBlock());
186 }
187 }
188 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
189 };
190 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
191 const VarDecl *ThreadIDVar,
192 const RegionCodeGenTy &CodeGen,
193 OpenMPDirectiveKind Kind, bool HasCancel,
194 const UntiedTaskActionTy &Action)
195 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
196 ThreadIDVar(ThreadIDVar), Action(Action) {
197 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 197, __PRETTY_FUNCTION__))
;
198 }
199
200 /// Get a variable or parameter for storing global thread id
201 /// inside OpenMP construct.
202 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
203
204 /// Get an LValue for the current ThreadID variable.
205 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
206
207 /// Get the name of the capture helper.
208 StringRef getHelperName() const override { return ".omp_outlined."; }
209
210 void emitUntiedSwitch(CodeGenFunction &CGF) override {
211 Action.emitUntiedSwitch(CGF);
212 }
213
214 static bool classof(const CGCapturedStmtInfo *Info) {
215 return CGOpenMPRegionInfo::classof(Info) &&
216 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
217 TaskOutlinedRegion;
218 }
219
220private:
221 /// A variable or parameter storing global thread id for OpenMP
222 /// constructs.
223 const VarDecl *ThreadIDVar;
224 /// Action for emitting code for untied tasks.
225 const UntiedTaskActionTy &Action;
226};
227
228/// API for inlined captured statement code generation in OpenMP
229/// constructs.
230class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
231public:
232 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
233 const RegionCodeGenTy &CodeGen,
234 OpenMPDirectiveKind Kind, bool HasCancel)
235 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
236 OldCSI(OldCSI),
237 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
238
239 // Retrieve the value of the context parameter.
240 llvm::Value *getContextValue() const override {
241 if (OuterRegionInfo)
242 return OuterRegionInfo->getContextValue();
243 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 243)
;
244 }
245
246 void setContextValue(llvm::Value *V) override {
247 if (OuterRegionInfo) {
248 OuterRegionInfo->setContextValue(V);
249 return;
250 }
251 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 251)
;
252 }
253
254 /// Lookup the captured field decl for a variable.
255 const FieldDecl *lookup(const VarDecl *VD) const override {
256 if (OuterRegionInfo)
257 return OuterRegionInfo->lookup(VD);
258 // If there is no outer outlined region,no need to lookup in a list of
259 // captured variables, we can use the original one.
260 return nullptr;
261 }
262
263 FieldDecl *getThisFieldDecl() const override {
264 if (OuterRegionInfo)
265 return OuterRegionInfo->getThisFieldDecl();
266 return nullptr;
267 }
268
269 /// Get a variable or parameter for storing global thread id
270 /// inside OpenMP construct.
271 const VarDecl *getThreadIDVariable() const override {
272 if (OuterRegionInfo)
273 return OuterRegionInfo->getThreadIDVariable();
274 return nullptr;
275 }
276
277 /// Get an LValue for the current ThreadID variable.
278 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
279 if (OuterRegionInfo)
280 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
281 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 281)
;
282 }
283
284 /// Get the name of the capture helper.
285 StringRef getHelperName() const override {
286 if (auto *OuterRegionInfo = getOldCSI())
287 return OuterRegionInfo->getHelperName();
288 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 288)
;
289 }
290
291 void emitUntiedSwitch(CodeGenFunction &CGF) override {
292 if (OuterRegionInfo)
293 OuterRegionInfo->emitUntiedSwitch(CGF);
294 }
295
296 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
297
298 static bool classof(const CGCapturedStmtInfo *Info) {
299 return CGOpenMPRegionInfo::classof(Info) &&
300 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
301 }
302
303 ~CGOpenMPInlinedRegionInfo() override = default;
304
305private:
306 /// CodeGen info about outer OpenMP region.
307 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
308 CGOpenMPRegionInfo *OuterRegionInfo;
309};
310
311/// API for captured statement code generation in OpenMP target
312/// constructs. For this captures, implicit parameters are used instead of the
313/// captured fields. The name of the target region has to be unique in a given
314/// application so it is provided by the client, because only the client has
315/// the information to generate that.
316class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
317public:
318 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
319 const RegionCodeGenTy &CodeGen, StringRef HelperName)
320 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
321 /*HasCancel=*/false),
322 HelperName(HelperName) {}
323
324 /// This is unused for target regions because each starts executing
325 /// with a single thread.
326 const VarDecl *getThreadIDVariable() const override { return nullptr; }
327
328 /// Get the name of the capture helper.
329 StringRef getHelperName() const override { return HelperName; }
330
331 static bool classof(const CGCapturedStmtInfo *Info) {
332 return CGOpenMPRegionInfo::classof(Info) &&
333 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
334 }
335
336private:
337 StringRef HelperName;
338};
339
340static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
341 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 341)
;
342}
343/// API for generation of expressions captured in a innermost OpenMP
344/// region.
345class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
346public:
347 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
348 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
349 OMPD_unknown,
350 /*HasCancel=*/false),
351 PrivScope(CGF) {
352 // Make sure the globals captured in the provided statement are local by
353 // using the privatization logic. We assume the same variable is not
354 // captured more than once.
355 for (const auto &C : CS.captures()) {
356 if (!C.capturesVariable() && !C.capturesVariableByCopy())
357 continue;
358
359 const VarDecl *VD = C.getCapturedVar();
360 if (VD->isLocalVarDeclOrParm())
361 continue;
362
363 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
364 /*RefersToEnclosingVariableOrCapture=*/false,
365 VD->getType().getNonReferenceType(), VK_LValue,
366 C.getLocation());
367 PrivScope.addPrivate(
368 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
369 }
370 (void)PrivScope.Privatize();
371 }
372
373 /// Lookup the captured field decl for a variable.
374 const FieldDecl *lookup(const VarDecl *VD) const override {
375 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
376 return FD;
377 return nullptr;
378 }
379
380 /// Emit the captured statement body.
381 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
382 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 382)
;
383 }
384
385 /// Get a variable or parameter for storing global thread id
386 /// inside OpenMP construct.
387 const VarDecl *getThreadIDVariable() const override {
388 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 388)
;
389 }
390
391 /// Get the name of the capture helper.
392 StringRef getHelperName() const override {
393 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 393)
;
394 }
395
396 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
397
398private:
399 /// Private scope to capture global variables.
400 CodeGenFunction::OMPPrivateScope PrivScope;
401};
402
403/// RAII for emitting code of OpenMP constructs.
404class InlinedOpenMPRegionRAII {
405 CodeGenFunction &CGF;
406 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
407 FieldDecl *LambdaThisCaptureField = nullptr;
408 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
409
410public:
411 /// Constructs region for combined constructs.
412 /// \param CodeGen Code generation sequence for combined directives. Includes
413 /// a list of functions used for code generation of implicitly inlined
414 /// regions.
415 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
416 OpenMPDirectiveKind Kind, bool HasCancel)
417 : CGF(CGF) {
418 // Start emission for the construct.
419 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
420 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
421 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
422 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
423 CGF.LambdaThisCaptureField = nullptr;
424 BlockInfo = CGF.BlockInfo;
425 CGF.BlockInfo = nullptr;
426 }
427
428 ~InlinedOpenMPRegionRAII() {
429 // Restore original CapturedStmtInfo only if we're done with code emission.
430 auto *OldCSI =
431 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
432 delete CGF.CapturedStmtInfo;
433 CGF.CapturedStmtInfo = OldCSI;
434 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
435 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
436 CGF.BlockInfo = BlockInfo;
437 }
438};
439
440/// Values for bit flags used in the ident_t to describe the fields.
441/// All enumeric elements are named and described in accordance with the code
442/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
443enum OpenMPLocationFlags : unsigned {
444 /// Use trampoline for internal microtask.
445 OMP_IDENT_IMD = 0x01,
446 /// Use c-style ident structure.
447 OMP_IDENT_KMPC = 0x02,
448 /// Atomic reduction option for kmpc_reduce.
449 OMP_ATOMIC_REDUCE = 0x10,
450 /// Explicit 'barrier' directive.
451 OMP_IDENT_BARRIER_EXPL = 0x20,
452 /// Implicit barrier in code.
453 OMP_IDENT_BARRIER_IMPL = 0x40,
454 /// Implicit barrier in 'for' directive.
455 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
456 /// Implicit barrier in 'sections' directive.
457 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
458 /// Implicit barrier in 'single' directive.
459 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
460 /// Call of __kmp_for_static_init for static loop.
461 OMP_IDENT_WORK_LOOP = 0x200,
462 /// Call of __kmp_for_static_init for sections.
463 OMP_IDENT_WORK_SECTIONS = 0x400,
464 /// Call of __kmp_for_static_init for distribute.
465 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
466 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
467};
468
469namespace {
470LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
471/// Values for bit flags for marking which requires clauses have been used.
472enum OpenMPOffloadingRequiresDirFlags : int64_t {
473 /// flag undefined.
474 OMP_REQ_UNDEFINED = 0x000,
475 /// no requires clause present.
476 OMP_REQ_NONE = 0x001,
477 /// reverse_offload clause.
478 OMP_REQ_REVERSE_OFFLOAD = 0x002,
479 /// unified_address clause.
480 OMP_REQ_UNIFIED_ADDRESS = 0x004,
481 /// unified_shared_memory clause.
482 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
483 /// dynamic_allocators clause.
484 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
485 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
486};
487
488enum OpenMPOffloadingReservedDeviceIDs {
489 /// Device ID if the device was not defined, runtime should get it
490 /// from environment variables in the spec.
491 OMP_DEVICEID_UNDEF = -1,
492};
493} // anonymous namespace
494
495/// Describes ident structure that describes a source location.
496/// All descriptions are taken from
497/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
498/// Original structure:
499/// typedef struct ident {
500/// kmp_int32 reserved_1; /**< might be used in Fortran;
501/// see above */
502/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
503/// KMP_IDENT_KMPC identifies this union
504/// member */
505/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
506/// see above */
507///#if USE_ITT_BUILD
508/// /* but currently used for storing
509/// region-specific ITT */
510/// /* contextual information. */
511///#endif /* USE_ITT_BUILD */
512/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
513/// C++ */
514/// char const *psource; /**< String describing the source location.
515/// The string is composed of semi-colon separated
516// fields which describe the source file,
517/// the function and a pair of line numbers that
518/// delimit the construct.
519/// */
520/// } ident_t;
521enum IdentFieldIndex {
522 /// might be used in Fortran
523 IdentField_Reserved_1,
524 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
525 IdentField_Flags,
526 /// Not really used in Fortran any more
527 IdentField_Reserved_2,
528 /// Source[4] in Fortran, do not use for C++
529 IdentField_Reserved_3,
530 /// String describing the source location. The string is composed of
531 /// semi-colon separated fields which describe the source file, the function
532 /// and a pair of line numbers that delimit the construct.
533 IdentField_PSource
534};
535
536/// Schedule types for 'omp for' loops (these enumerators are taken from
537/// the enum sched_type in kmp.h).
538enum OpenMPSchedType {
539 /// Lower bound for default (unordered) versions.
540 OMP_sch_lower = 32,
541 OMP_sch_static_chunked = 33,
542 OMP_sch_static = 34,
543 OMP_sch_dynamic_chunked = 35,
544 OMP_sch_guided_chunked = 36,
545 OMP_sch_runtime = 37,
546 OMP_sch_auto = 38,
547 /// static with chunk adjustment (e.g., simd)
548 OMP_sch_static_balanced_chunked = 45,
549 /// Lower bound for 'ordered' versions.
550 OMP_ord_lower = 64,
551 OMP_ord_static_chunked = 65,
552 OMP_ord_static = 66,
553 OMP_ord_dynamic_chunked = 67,
554 OMP_ord_guided_chunked = 68,
555 OMP_ord_runtime = 69,
556 OMP_ord_auto = 70,
557 OMP_sch_default = OMP_sch_static,
558 /// dist_schedule types
559 OMP_dist_sch_static_chunked = 91,
560 OMP_dist_sch_static = 92,
561 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
562 /// Set if the monotonic schedule modifier was present.
563 OMP_sch_modifier_monotonic = (1 << 29),
564 /// Set if the nonmonotonic schedule modifier was present.
565 OMP_sch_modifier_nonmonotonic = (1 << 30),
566};
567
568enum OpenMPRTLFunction {
569 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
570 /// kmpc_micro microtask, ...);
571 OMPRTL__kmpc_fork_call,
572 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
573 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
574 OMPRTL__kmpc_threadprivate_cached,
575 /// Call to void __kmpc_threadprivate_register( ident_t *,
576 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
577 OMPRTL__kmpc_threadprivate_register,
578 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
579 OMPRTL__kmpc_global_thread_num,
580 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
581 // kmp_critical_name *crit);
582 OMPRTL__kmpc_critical,
583 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
584 // global_tid, kmp_critical_name *crit, uintptr_t hint);
585 OMPRTL__kmpc_critical_with_hint,
586 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
587 // kmp_critical_name *crit);
588 OMPRTL__kmpc_end_critical,
589 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
590 // global_tid);
591 OMPRTL__kmpc_cancel_barrier,
592 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
593 OMPRTL__kmpc_barrier,
594 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
595 OMPRTL__kmpc_for_static_fini,
596 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
597 // global_tid);
598 OMPRTL__kmpc_serialized_parallel,
599 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
600 // global_tid);
601 OMPRTL__kmpc_end_serialized_parallel,
602 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
603 // kmp_int32 num_threads);
604 OMPRTL__kmpc_push_num_threads,
605 // Call to void __kmpc_flush(ident_t *loc);
606 OMPRTL__kmpc_flush,
607 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
608 OMPRTL__kmpc_master,
609 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
610 OMPRTL__kmpc_end_master,
611 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
612 // int end_part);
613 OMPRTL__kmpc_omp_taskyield,
614 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
615 OMPRTL__kmpc_single,
616 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
617 OMPRTL__kmpc_end_single,
618 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
619 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
620 // kmp_routine_entry_t *task_entry);
621 OMPRTL__kmpc_omp_task_alloc,
622 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
623 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
624 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
625 // kmp_int64 device_id);
626 OMPRTL__kmpc_omp_target_task_alloc,
627 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
628 // new_task);
629 OMPRTL__kmpc_omp_task,
630 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
631 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
632 // kmp_int32 didit);
633 OMPRTL__kmpc_copyprivate,
634 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
635 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
636 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
637 OMPRTL__kmpc_reduce,
638 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
639 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
640 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
641 // *lck);
642 OMPRTL__kmpc_reduce_nowait,
643 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
644 // kmp_critical_name *lck);
645 OMPRTL__kmpc_end_reduce,
646 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
647 // kmp_critical_name *lck);
648 OMPRTL__kmpc_end_reduce_nowait,
649 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
650 // kmp_task_t * new_task);
651 OMPRTL__kmpc_omp_task_begin_if0,
652 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
653 // kmp_task_t * new_task);
654 OMPRTL__kmpc_omp_task_complete_if0,
655 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
656 OMPRTL__kmpc_ordered,
657 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
658 OMPRTL__kmpc_end_ordered,
659 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
660 // global_tid);
661 OMPRTL__kmpc_omp_taskwait,
662 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
663 OMPRTL__kmpc_taskgroup,
664 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
665 OMPRTL__kmpc_end_taskgroup,
666 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
667 // int proc_bind);
668 OMPRTL__kmpc_push_proc_bind,
669 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
670 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
671 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
672 OMPRTL__kmpc_omp_task_with_deps,
673 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
674 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
675 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
676 OMPRTL__kmpc_omp_wait_deps,
677 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
678 // global_tid, kmp_int32 cncl_kind);
679 OMPRTL__kmpc_cancellationpoint,
680 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
681 // kmp_int32 cncl_kind);
682 OMPRTL__kmpc_cancel,
683 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
684 // kmp_int32 num_teams, kmp_int32 thread_limit);
685 OMPRTL__kmpc_push_num_teams,
686 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
687 // microtask, ...);
688 OMPRTL__kmpc_fork_teams,
689 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
690 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
691 // sched, kmp_uint64 grainsize, void *task_dup);
692 OMPRTL__kmpc_taskloop,
693 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
694 // num_dims, struct kmp_dim *dims);
695 OMPRTL__kmpc_doacross_init,
696 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
697 OMPRTL__kmpc_doacross_fini,
698 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
699 // *vec);
700 OMPRTL__kmpc_doacross_post,
701 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
702 // *vec);
703 OMPRTL__kmpc_doacross_wait,
704 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
705 // *data);
706 OMPRTL__kmpc_task_reduction_init,
707 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
708 // *d);
709 OMPRTL__kmpc_task_reduction_get_th_data,
710 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
711 OMPRTL__kmpc_alloc,
712 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
713 OMPRTL__kmpc_free,
714
715 //
716 // Offloading related calls
717 //
718 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
719 // size);
720 OMPRTL__kmpc_push_target_tripcount,
721 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
722 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
723 // *arg_types);
724 OMPRTL__tgt_target,
725 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
726 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
727 // *arg_types);
728 OMPRTL__tgt_target_nowait,
729 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
730 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
731 // *arg_types, int32_t num_teams, int32_t thread_limit);
732 OMPRTL__tgt_target_teams,
733 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
734 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
735 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
736 OMPRTL__tgt_target_teams_nowait,
737 // Call to void __tgt_register_requires(int64_t flags);
738 OMPRTL__tgt_register_requires,
739 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
740 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
741 OMPRTL__tgt_target_data_begin,
742 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
743 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
744 // *arg_types);
745 OMPRTL__tgt_target_data_begin_nowait,
746 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
747 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
748 OMPRTL__tgt_target_data_end,
749 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
750 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
751 // *arg_types);
752 OMPRTL__tgt_target_data_end_nowait,
753 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
754 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
755 OMPRTL__tgt_target_data_update,
756 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
757 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
758 // *arg_types);
759 OMPRTL__tgt_target_data_update_nowait,
760 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
761 OMPRTL__tgt_mapper_num_components,
762 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
763 // *base, void *begin, int64_t size, int64_t type);
764 OMPRTL__tgt_push_mapper_component,
765};
766
767/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
768/// region.
769class CleanupTy final : public EHScopeStack::Cleanup {
770 PrePostActionTy *Action;
771
772public:
773 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
774 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
775 if (!CGF.HaveInsertPoint())
776 return;
777 Action->Exit(CGF);
778 }
779};
780
781} // anonymous namespace
782
783void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
784 CodeGenFunction::RunCleanupsScope Scope(CGF);
785 if (PrePostAction
11.1
Field 'PrePostAction' is null
11.1
Field 'PrePostAction' is null
) {
12
Taking false branch
786 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
787 Callback(CodeGen, CGF, *PrePostAction);
788 } else {
789 PrePostActionTy Action;
790 Callback(CodeGen, CGF, Action);
13
Calling 'RegionCodeGenTy::CallbackFn'
791 }
792}
793
794/// Check if the combiner is a call to UDR combiner and if it is so return the
795/// UDR decl used for reduction.
796static const OMPDeclareReductionDecl *
797getReductionInit(const Expr *ReductionOp) {
798 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
799 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
800 if (const auto *DRE =
801 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
802 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
803 return DRD;
804 return nullptr;
805}
806
807static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
808 const OMPDeclareReductionDecl *DRD,
809 const Expr *InitOp,
810 Address Private, Address Original,
811 QualType Ty) {
812 if (DRD->getInitializer()) {
813 std::pair<llvm::Function *, llvm::Function *> Reduction =
814 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
815 const auto *CE = cast<CallExpr>(InitOp);
816 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
817 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
818 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
819 const auto *LHSDRE =
820 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
821 const auto *RHSDRE =
822 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
823 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
824 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
825 [=]() { return Private; });
826 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
827 [=]() { return Original; });
828 (void)PrivateScope.Privatize();
829 RValue Func = RValue::get(Reduction.second);
830 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
831 CGF.EmitIgnoredExpr(InitOp);
832 } else {
833 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
834 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
835 auto *GV = new llvm::GlobalVariable(
836 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
837 llvm::GlobalValue::PrivateLinkage, Init, Name);
838 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
839 RValue InitRVal;
840 switch (CGF.getEvaluationKind(Ty)) {
841 case TEK_Scalar:
842 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
843 break;
844 case TEK_Complex:
845 InitRVal =
846 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
847 break;
848 case TEK_Aggregate:
849 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
850 break;
851 }
852 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
853 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
854 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
855 /*IsInitializer=*/false);
856 }
857}
858
859/// Emit initialization of arrays of complex types.
860/// \param DestAddr Address of the array.
861/// \param Type Type of array.
862/// \param Init Initial expression of array.
863/// \param SrcAddr Address of the original array.
864static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
865 QualType Type, bool EmitDeclareReductionInit,
866 const Expr *Init,
867 const OMPDeclareReductionDecl *DRD,
868 Address SrcAddr = Address::invalid()) {
869 // Perform element-by-element initialization.
870 QualType ElementTy;
871
872 // Drill down to the base element type on both arrays.
873 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
874 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
875 DestAddr =
876 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
877 if (DRD)
878 SrcAddr =
879 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
880
881 llvm::Value *SrcBegin = nullptr;
882 if (DRD)
883 SrcBegin = SrcAddr.getPointer();
884 llvm::Value *DestBegin = DestAddr.getPointer();
885 // Cast from pointer to array type to pointer to single element.
886 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
887 // The basic structure here is a while-do loop.
888 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
889 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
890 llvm::Value *IsEmpty =
891 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
892 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
893
894 // Enter the loop body, making that address the current address.
895 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
896 CGF.EmitBlock(BodyBB);
897
898 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
899
900 llvm::PHINode *SrcElementPHI = nullptr;
901 Address SrcElementCurrent = Address::invalid();
902 if (DRD) {
903 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
904 "omp.arraycpy.srcElementPast");
905 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
906 SrcElementCurrent =
907 Address(SrcElementPHI,
908 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
909 }
910 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
911 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
912 DestElementPHI->addIncoming(DestBegin, EntryBB);
913 Address DestElementCurrent =
914 Address(DestElementPHI,
915 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
916
917 // Emit copy.
918 {
919 CodeGenFunction::RunCleanupsScope InitScope(CGF);
920 if (EmitDeclareReductionInit) {
921 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
922 SrcElementCurrent, ElementTy);
923 } else
924 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
925 /*IsInitializer=*/false);
926 }
927
928 if (DRD) {
929 // Shift the address forward by one element.
930 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
931 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
932 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
933 }
934
935 // Shift the address forward by one element.
936 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
937 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
938 // Check whether we've reached the end.
939 llvm::Value *Done =
940 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
941 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
942 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
943
944 // Done.
945 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
946}
947
948LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
949 return CGF.EmitOMPSharedLValue(E);
950}
951
952LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
953 const Expr *E) {
954 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
955 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
956 return LValue();
957}
958
959void ReductionCodeGen::emitAggregateInitialization(
960 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
961 const OMPDeclareReductionDecl *DRD) {
962 // Emit VarDecl with copy init for arrays.
963 // Get the address of the original variable captured in current
964 // captured region.
965 const auto *PrivateVD =
966 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
967 bool EmitDeclareReductionInit =
968 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
969 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
970 EmitDeclareReductionInit,
971 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
972 : PrivateVD->getInit(),
973 DRD, SharedLVal.getAddress(CGF));
974}
975
976ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
977 ArrayRef<const Expr *> Privates,
978 ArrayRef<const Expr *> ReductionOps) {
979 ClausesData.reserve(Shareds.size());
980 SharedAddresses.reserve(Shareds.size());
981 Sizes.reserve(Shareds.size());
982 BaseDecls.reserve(Shareds.size());
983 auto IPriv = Privates.begin();
984 auto IRed = ReductionOps.begin();
985 for (const Expr *Ref : Shareds) {
986 ClausesData.emplace_back(Ref, *IPriv, *IRed);
987 std::advance(IPriv, 1);
988 std::advance(IRed, 1);
989 }
990}
991
992void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
993 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 994, __PRETTY_FUNCTION__))
994 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 994, __PRETTY_FUNCTION__))
;
995 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
996 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
997 SharedAddresses.emplace_back(First, Second);
998}
999
1000void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
1001 const auto *PrivateVD =
1002 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1003 QualType PrivateType = PrivateVD->getType();
1004 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1005 if (!PrivateType->isVariablyModifiedType()) {
1006 Sizes.emplace_back(
1007 CGF.getTypeSize(
1008 SharedAddresses[N].first.getType().getNonReferenceType()),
1009 nullptr);
1010 return;
1011 }
1012 llvm::Value *Size;
1013 llvm::Value *SizeInChars;
1014 auto *ElemType = cast<llvm::PointerType>(
1015 SharedAddresses[N].first.getPointer(CGF)->getType())
1016 ->getElementType();
1017 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1018 if (AsArraySection) {
1019 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1020 SharedAddresses[N].first.getPointer(CGF));
1021 Size = CGF.Builder.CreateNUWAdd(
1022 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1023 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1024 } else {
1025 SizeInChars = CGF.getTypeSize(
1026 SharedAddresses[N].first.getType().getNonReferenceType());
1027 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1028 }
1029 Sizes.emplace_back(SizeInChars, Size);
1030 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1031 CGF,
1032 cast<OpaqueValueExpr>(
1033 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1034 RValue::get(Size));
1035 CGF.EmitVariablyModifiedType(PrivateType);
1036}
1037
1038void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1039 llvm::Value *Size) {
1040 const auto *PrivateVD =
1041 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1042 QualType PrivateType = PrivateVD->getType();
1043 if (!PrivateType->isVariablyModifiedType()) {
1044 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1046, __PRETTY_FUNCTION__))
1045 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1046, __PRETTY_FUNCTION__))
1046 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1046, __PRETTY_FUNCTION__))
;
1047 return;
1048 }
1049 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1050 CGF,
1051 cast<OpaqueValueExpr>(
1052 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1053 RValue::get(Size));
1054 CGF.EmitVariablyModifiedType(PrivateType);
1055}
1056
1057void ReductionCodeGen::emitInitialization(
1058 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1059 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1060 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1060, __PRETTY_FUNCTION__))
;
1061 const auto *PrivateVD =
1062 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1063 const OMPDeclareReductionDecl *DRD =
1064 getReductionInit(ClausesData[N].ReductionOp);
1065 QualType PrivateType = PrivateVD->getType();
1066 PrivateAddr = CGF.Builder.CreateElementBitCast(
1067 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1068 QualType SharedType = SharedAddresses[N].first.getType();
1069 SharedLVal = CGF.MakeAddrLValue(
1070 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
1071 CGF.ConvertTypeForMem(SharedType)),
1072 SharedType, SharedAddresses[N].first.getBaseInfo(),
1073 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1074 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1075 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1076 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1077 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1078 PrivateAddr, SharedLVal.getAddress(CGF),
1079 SharedLVal.getType());
1080 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1081 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1082 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1083 PrivateVD->getType().getQualifiers(),
1084 /*IsInitializer=*/false);
1085 }
1086}
1087
1088bool ReductionCodeGen::needCleanups(unsigned N) {
1089 const auto *PrivateVD =
1090 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1091 QualType PrivateType = PrivateVD->getType();
1092 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1093 return DTorKind != QualType::DK_none;
1094}
1095
1096void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1097 Address PrivateAddr) {
1098 const auto *PrivateVD =
1099 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1100 QualType PrivateType = PrivateVD->getType();
1101 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1102 if (needCleanups(N)) {
1103 PrivateAddr = CGF.Builder.CreateElementBitCast(
1104 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1105 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1106 }
1107}
1108
1109static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1110 LValue BaseLV) {
1111 BaseTy = BaseTy.getNonReferenceType();
1112 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1113 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1114 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1115 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
1116 } else {
1117 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
1118 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1119 }
1120 BaseTy = BaseTy->getPointeeType();
1121 }
1122 return CGF.MakeAddrLValue(
1123 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
1124 CGF.ConvertTypeForMem(ElTy)),
1125 BaseLV.getType(), BaseLV.getBaseInfo(),
1126 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1127}
1128
1129static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1130 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1131 llvm::Value *Addr) {
1132 Address Tmp = Address::invalid();
1133 Address TopTmp = Address::invalid();
1134 Address MostTopTmp = Address::invalid();
1135 BaseTy = BaseTy.getNonReferenceType();
1136 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1137 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1138 Tmp = CGF.CreateMemTemp(BaseTy);
1139 if (TopTmp.isValid())
1140 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1141 else
1142 MostTopTmp = Tmp;
1143 TopTmp = Tmp;
1144 BaseTy = BaseTy->getPointeeType();
1145 }
1146 llvm::Type *Ty = BaseLVType;
1147 if (Tmp.isValid())
1148 Ty = Tmp.getElementType();
1149 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1150 if (Tmp.isValid()) {
1151 CGF.Builder.CreateStore(Addr, Tmp);
1152 return MostTopTmp;
1153 }
1154 return Address(Addr, BaseLVAlignment);
1155}
1156
1157static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1158 const VarDecl *OrigVD = nullptr;
1159 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1160 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1161 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1162 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1163 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1164 Base = TempASE->getBase()->IgnoreParenImpCasts();
1165 DE = cast<DeclRefExpr>(Base);
1166 OrigVD = cast<VarDecl>(DE->getDecl());
1167 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1168 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1169 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1170 Base = TempASE->getBase()->IgnoreParenImpCasts();
1171 DE = cast<DeclRefExpr>(Base);
1172 OrigVD = cast<VarDecl>(DE->getDecl());
1173 }
1174 return OrigVD;
1175}
1176
1177Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1178 Address PrivateAddr) {
1179 const DeclRefExpr *DE;
1180 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1181 BaseDecls.emplace_back(OrigVD);
1182 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1183 LValue BaseLValue =
1184 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1185 OriginalBaseLValue);
1186 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1187 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1188 llvm::Value *PrivatePointer =
1189 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1190 PrivateAddr.getPointer(),
1191 SharedAddresses[N].first.getAddress(CGF).getType());
1192 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1193 return castToBase(CGF, OrigVD->getType(),
1194 SharedAddresses[N].first.getType(),
1195 OriginalBaseLValue.getAddress(CGF).getType(),
1196 OriginalBaseLValue.getAlignment(), Ptr);
1197 }
1198 BaseDecls.emplace_back(
1199 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1200 return PrivateAddr;
1201}
1202
1203bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1204 const OMPDeclareReductionDecl *DRD =
1205 getReductionInit(ClausesData[N].ReductionOp);
1206 return DRD && DRD->getInitializer();
1207}
1208
1209LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1210 return CGF.EmitLoadOfPointerLValue(
1211 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1212 getThreadIDVariable()->getType()->castAs<PointerType>());
1213}
1214
1215void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1216 if (!CGF.HaveInsertPoint())
1217 return;
1218 // 1.2.2 OpenMP Language Terminology
1219 // Structured block - An executable statement with a single entry at the
1220 // top and a single exit at the bottom.
1221 // The point of exit cannot be a branch out of the structured block.
1222 // longjmp() and throw() must not violate the entry/exit criteria.
1223 CGF.EHStack.pushTerminate();
1224 CodeGen(CGF);
1225 CGF.EHStack.popTerminate();
1226}
1227
1228LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1229 CodeGenFunction &CGF) {
1230 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1231 getThreadIDVariable()->getType(),
1232 AlignmentSource::Decl);
1233}
1234
1235static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1236 QualType FieldTy) {
1237 auto *Field = FieldDecl::Create(
1238 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1239 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1240 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1241 Field->setAccess(AS_public);
1242 DC->addDecl(Field);
1243 return Field;
1244}
1245
1246CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1247 StringRef Separator)
1248 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1249 OffloadEntriesInfoManager(CGM) {
1250 ASTContext &C = CGM.getContext();
1251 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1252 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1253 RD->startDefinition();
1254 // reserved_1
1255 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1256 // flags
1257 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1258 // reserved_2
1259 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1260 // reserved_3
1261 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1262 // psource
1263 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1264 RD->completeDefinition();
1265 IdentQTy = C.getRecordType(RD);
1266 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1267 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1268
1269 loadOffloadInfoMetadata();
1270}
1271
1272bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1273 const GlobalDecl &OldGD,
1274 llvm::GlobalValue *OrigAddr,
1275 bool IsForDefinition) {
1276 // Emit at least a definition for the aliasee if the the address of the
1277 // original function is requested.
1278 if (IsForDefinition || OrigAddr)
1279 (void)CGM.GetAddrOfGlobal(NewGD);
1280 StringRef NewMangledName = CGM.getMangledName(NewGD);
1281 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1282 if (Addr && !Addr->isDeclaration()) {
1283 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1284 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(NewGD);
1285 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1286
1287 // Create a reference to the named value. This ensures that it is emitted
1288 // if a deferred decl.
1289 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1290
1291 // Create the new alias itself, but don't set a name yet.
1292 auto *GA =
1293 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1294
1295 if (OrigAddr) {
1296 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1296, __PRETTY_FUNCTION__))
;
1297
1298 GA->takeName(OrigAddr);
1299 OrigAddr->replaceAllUsesWith(
1300 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1301 OrigAddr->eraseFromParent();
1302 } else {
1303 GA->setName(CGM.getMangledName(OldGD));
1304 }
1305
1306 // Set attributes which are particular to an alias; this is a
1307 // specialization of the attributes which may be set on a global function.
1308 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1309 D->isWeakImported())
1310 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1311
1312 CGM.SetCommonAttributes(OldGD, GA);
1313 return true;
1314 }
1315 return false;
1316}
1317
1318void CGOpenMPRuntime::clear() {
1319 InternalVars.clear();
1320 // Clean non-target variable declarations possibly used only in debug info.
1321 for (const auto &Data : EmittedNonTargetVariables) {
1322 if (!Data.getValue().pointsToAliveValue())
1323 continue;
1324 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1325 if (!GV)
1326 continue;
1327 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1328 continue;
1329 GV->eraseFromParent();
1330 }
1331 // Emit aliases for the deferred aliasees.
1332 for (const auto &Pair : DeferredVariantFunction) {
1333 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1334 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1335 // If not able to emit alias, just emit original declaration.
1336 (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1337 /*IsForDefinition=*/false);
1338 }
1339}
1340
1341std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1342 SmallString<128> Buffer;
1343 llvm::raw_svector_ostream OS(Buffer);
1344 StringRef Sep = FirstSeparator;
1345 for (StringRef Part : Parts) {
1346 OS << Sep << Part;
1347 Sep = Separator;
1348 }
1349 return std::string(OS.str());
1350}
1351
1352static llvm::Function *
1353emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1354 const Expr *CombinerInitializer, const VarDecl *In,
1355 const VarDecl *Out, bool IsCombiner) {
1356 // void .omp_combiner.(Ty *in, Ty *out);
1357 ASTContext &C = CGM.getContext();
1358 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1359 FunctionArgList Args;
1360 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1361 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1362 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1363 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1364 Args.push_back(&OmpOutParm);
1365 Args.push_back(&OmpInParm);
1366 const CGFunctionInfo &FnInfo =
1367 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1368 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1369 std::string Name = CGM.getOpenMPRuntime().getName(
1370 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1371 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1372 Name, &CGM.getModule());
1373 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1374 if (CGM.getLangOpts().Optimize) {
1375 Fn->removeFnAttr(llvm::Attribute::NoInline);
1376 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1377 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1378 }
1379 CodeGenFunction CGF(CGM);
1380 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1381 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1382 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1383 Out->getLocation());
1384 CodeGenFunction::OMPPrivateScope Scope(CGF);
1385 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1386 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1387 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1388 .getAddress(CGF);
1389 });
1390 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1391 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1392 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1393 .getAddress(CGF);
1394 });
1395 (void)Scope.Privatize();
1396 if (!IsCombiner && Out->hasInit() &&
1397 !CGF.isTrivialInitializer(Out->getInit())) {
1398 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1399 Out->getType().getQualifiers(),
1400 /*IsInitializer=*/true);
1401 }
1402 if (CombinerInitializer)
1403 CGF.EmitIgnoredExpr(CombinerInitializer);
1404 Scope.ForceCleanup();
1405 CGF.FinishFunction();
1406 return Fn;
1407}
1408
1409void CGOpenMPRuntime::emitUserDefinedReduction(
1410 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1411 if (UDRMap.count(D) > 0)
1412 return;
1413 llvm::Function *Combiner = emitCombinerOrInitializer(
1414 CGM, D->getType(), D->getCombiner(),
1415 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1416 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1417 /*IsCombiner=*/true);
1418 llvm::Function *Initializer = nullptr;
1419 if (const Expr *Init = D->getInitializer()) {
1420 Initializer = emitCombinerOrInitializer(
1421 CGM, D->getType(),
1422 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1423 : nullptr,
1424 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1425 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1426 /*IsCombiner=*/false);
1427 }
1428 UDRMap.try_emplace(D, Combiner, Initializer);
1429 if (CGF) {
1430 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1431 Decls.second.push_back(D);
1432 }
1433}
1434
1435std::pair<llvm::Function *, llvm::Function *>
1436CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1437 auto I = UDRMap.find(D);
1438 if (I != UDRMap.end())
1439 return I->second;
1440 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1441 return UDRMap.lookup(D);
1442}
1443
1444namespace {
1445// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1446// Builder if one is present.
1447struct PushAndPopStackRAII {
1448 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1449 bool HasCancel)
1450 : OMPBuilder(OMPBuilder) {
1451 if (!OMPBuilder)
1452 return;
1453
1454 // The following callback is the crucial part of clangs cleanup process.
1455 //
1456 // NOTE:
1457 // Once the OpenMPIRBuilder is used to create parallel regions (and
1458 // similar), the cancellation destination (Dest below) is determined via
1459 // IP. That means if we have variables to finalize we split the block at IP,
1460 // use the new block (=BB) as destination to build a JumpDest (via
1461 // getJumpDestInCurrentScope(BB)) which then is fed to
1462 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1463 // to push & pop an FinalizationInfo object.
1464 // The FiniCB will still be needed but at the point where the
1465 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1466 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1467 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1468, __PRETTY_FUNCTION__))
1468 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1468, __PRETTY_FUNCTION__))
;
1469 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1470 CGF.Builder.restoreIP(IP);
1471 CodeGenFunction::JumpDest Dest =
1472 CGF.getOMPCancelDestination(OMPD_parallel);
1473 CGF.EmitBranchThroughCleanup(Dest);
1474 };
1475
1476 // TODO: Remove this once we emit parallel regions through the
1477 // OpenMPIRBuilder as it can do this setup internally.
1478 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1479 {FiniCB, OMPD_parallel, HasCancel});
1480 OMPBuilder->pushFinalizationCB(std::move(FI));
1481 }
1482 ~PushAndPopStackRAII() {
1483 if (OMPBuilder)
1484 OMPBuilder->popFinalizationCB();
1485 }
1486 llvm::OpenMPIRBuilder *OMPBuilder;
1487};
1488} // namespace
1489
1490static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1491 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1492 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1493 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1494 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1495, __PRETTY_FUNCTION__))
1495 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1495, __PRETTY_FUNCTION__))
;
1496 CodeGenFunction CGF(CGM, true);
1497 bool HasCancel = false;
1498 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1499 HasCancel = OPD->hasCancel();
1500 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1501 HasCancel = OPSD->hasCancel();
1502 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1503 HasCancel = OPFD->hasCancel();
1504 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1505 HasCancel = OPFD->hasCancel();
1506 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1507 HasCancel = OPFD->hasCancel();
1508 else if (const auto *OPFD =
1509 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1510 HasCancel = OPFD->hasCancel();
1511 else if (const auto *OPFD =
1512 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1513 HasCancel = OPFD->hasCancel();
1514
1515 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1516 // parallel region to make cancellation barriers work properly.
1517 llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder();
1518 PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
1519 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1520 HasCancel, OutlinedHelperName);
1521 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1522 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1523}
1524
1525llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1526 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1527 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1528 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1529 return emitParallelOrTeamsOutlinedFunction(
1530 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1531}
1532
1533llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1534 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1535 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1536 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1537 return emitParallelOrTeamsOutlinedFunction(
1538 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1539}
1540
1541llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1542 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1543 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1544 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1545 bool Tied, unsigned &NumberOfParts) {
1546 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1547 PrePostActionTy &) {
1548 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1549 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1550 llvm::Value *TaskArgs[] = {
1551 UpLoc, ThreadID,
1552 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1553 TaskTVar->getType()->castAs<PointerType>())
1554 .getPointer(CGF)};
1555 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1556 };
1557 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1558 UntiedCodeGen);
1559 CodeGen.setAction(Action);
1560 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1561, __PRETTY_FUNCTION__))
1561 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1561, __PRETTY_FUNCTION__))
;
1562 const OpenMPDirectiveKind Region =
1563 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1564 : OMPD_task;
1565 const CapturedStmt *CS = D.getCapturedStmt(Region);
1566 bool HasCancel = false;
1567 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1568 HasCancel = TD->hasCancel();
1569 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1570 HasCancel = TD->hasCancel();
1571 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1572 HasCancel = TD->hasCancel();
1573 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1574 HasCancel = TD->hasCancel();
1575
1576 CodeGenFunction CGF(CGM, true);
1577 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1578 InnermostKind, HasCancel, Action);
1579 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1580 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1581 if (!Tied)
1582 NumberOfParts = Action.getNumberOfParts();
1583 return Res;
1584}
1585
1586static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1587 const RecordDecl *RD, const CGRecordLayout &RL,
1588 ArrayRef<llvm::Constant *> Data) {
1589 llvm::StructType *StructTy = RL.getLLVMType();
1590 unsigned PrevIdx = 0;
1591 ConstantInitBuilder CIBuilder(CGM);
1592 auto DI = Data.begin();
1593 for (const FieldDecl *FD : RD->fields()) {
1594 unsigned Idx = RL.getLLVMFieldNo(FD);
1595 // Fill the alignment.
1596 for (unsigned I = PrevIdx; I < Idx; ++I)
1597 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1598 PrevIdx = Idx + 1;
1599 Fields.add(*DI);
1600 ++DI;
1601 }
1602}
1603
1604template <class... As>
1605static llvm::GlobalVariable *
1606createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1607 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1608 As &&... Args) {
1609 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1610 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1611 ConstantInitBuilder CIBuilder(CGM);
1612 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1613 buildStructValue(Fields, CGM, RD, RL, Data);
1614 return Fields.finishAndCreateGlobal(
1615 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1616 std::forward<As>(Args)...);
1617}
1618
1619template <typename T>
1620static void
1621createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1622 ArrayRef<llvm::Constant *> Data,
1623 T &Parent) {
1624 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1625 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1626 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1627 buildStructValue(Fields, CGM, RD, RL, Data);
1628 Fields.finishAndAddTo(Parent);
1629}
1630
1631Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1632 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1633 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1634 FlagsTy FlagsKey(Flags, Reserved2Flags);
1635 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1636 if (!Entry) {
1637 if (!DefaultOpenMPPSource) {
1638 // Initialize default location for psource field of ident_t structure of
1639 // all ident_t objects. Format is ";file;function;line;column;;".
1640 // Taken from
1641 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1642 DefaultOpenMPPSource =
1643 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1644 DefaultOpenMPPSource =
1645 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1646 }
1647
1648 llvm::Constant *Data[] = {
1649 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1650 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1651 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1652 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1653 llvm::GlobalValue *DefaultOpenMPLocation =
1654 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1655 llvm::GlobalValue::PrivateLinkage);
1656 DefaultOpenMPLocation->setUnnamedAddr(
1657 llvm::GlobalValue::UnnamedAddr::Global);
1658
1659 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1660 }
1661 return Address(Entry, Align);
1662}
1663
1664void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1665 bool AtCurrentPoint) {
1666 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1667 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1667, __PRETTY_FUNCTION__))
;
1668
1669 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1670 if (AtCurrentPoint) {
1671 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1672 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1673 } else {
1674 Elem.second.ServiceInsertPt =
1675 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1676 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1677 }
1678}
1679
1680void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1681 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1682 if (Elem.second.ServiceInsertPt) {
1683 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1684 Elem.second.ServiceInsertPt = nullptr;
1685 Ptr->eraseFromParent();
1686 }
1687}
1688
1689llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1690 SourceLocation Loc,
1691 unsigned Flags) {
1692 Flags |= OMP_IDENT_KMPC;
1693 // If no debug info is generated - return global default location.
1694 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1695 Loc.isInvalid())
1696 return getOrCreateDefaultLocation(Flags).getPointer();
1697
1698 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1698, __PRETTY_FUNCTION__))
;
1699
1700 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1701 Address LocValue = Address::invalid();
1702 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1703 if (I != OpenMPLocThreadIDMap.end())
1704 LocValue = Address(I->second.DebugLoc, Align);
1705
1706 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1707 // GetOpenMPThreadID was called before this routine.
1708 if (!LocValue.isValid()) {
1709 // Generate "ident_t .kmpc_loc.addr;"
1710 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1711 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1712 Elem.second.DebugLoc = AI.getPointer();
1713 LocValue = AI;
1714
1715 if (!Elem.second.ServiceInsertPt)
1716 setLocThreadIdInsertPt(CGF);
1717 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1718 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1719 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1720 CGF.getTypeSize(IdentQTy));
1721 }
1722
1723 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1724 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1725 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1726 LValue PSource =
1727 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1728
1729 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1730 if (OMPDebugLoc == nullptr) {
1731 SmallString<128> Buffer2;
1732 llvm::raw_svector_ostream OS2(Buffer2);
1733 // Build debug location
1734 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1735 OS2 << ";" << PLoc.getFilename() << ";";
1736 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1737 OS2 << FD->getQualifiedNameAsString();
1738 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1739 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1740 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1741 }
1742 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1743 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1744
1745 // Our callers always pass this to a runtime function, so for
1746 // convenience, go ahead and return a naked pointer.
1747 return LocValue.getPointer();
1748}
1749
1750llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1751 SourceLocation Loc) {
1752 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1752, __PRETTY_FUNCTION__))
;
1753
1754 llvm::Value *ThreadID = nullptr;
1755 // Check whether we've already cached a load of the thread id in this
1756 // function.
1757 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1758 if (I != OpenMPLocThreadIDMap.end()) {
1759 ThreadID = I->second.ThreadID;
1760 if (ThreadID != nullptr)
1761 return ThreadID;
1762 }
1763 // If exceptions are enabled, do not use parameter to avoid possible crash.
1764 if (auto *OMPRegionInfo =
1765 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1766 if (OMPRegionInfo->getThreadIDVariable()) {
1767 // Check if this an outlined function with thread id passed as argument.
1768 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1769 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1770 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1771 !CGF.getLangOpts().CXXExceptions ||
1772 CGF.Builder.GetInsertBlock() == TopBlock ||
1773 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1774 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1775 TopBlock ||
1776 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1777 CGF.Builder.GetInsertBlock()) {
1778 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1779 // If value loaded in entry block, cache it and use it everywhere in
1780 // function.
1781 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1782 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1783 Elem.second.ThreadID = ThreadID;
1784 }
1785 return ThreadID;
1786 }
1787 }
1788 }
1789
1790 // This is not an outlined function region - need to call __kmpc_int32
1791 // kmpc_global_thread_num(ident_t *loc).
1792 // Generate thread id value and cache this value for use across the
1793 // function.
1794 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1795 if (!Elem.second.ServiceInsertPt)
1796 setLocThreadIdInsertPt(CGF);
1797 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1798 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1799 llvm::CallInst *Call = CGF.Builder.CreateCall(
1800 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1801 emitUpdateLocation(CGF, Loc));
1802 Call->setCallingConv(CGF.getRuntimeCC());
1803 Elem.second.ThreadID = Call;
1804 return Call;
1805}
1806
1807void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1808 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1808, __PRETTY_FUNCTION__))
;
1809 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1810 clearLocThreadIdInsertPt(CGF);
1811 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1812 }
1813 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1814 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1815 UDRMap.erase(D);
1816 FunctionUDRMap.erase(CGF.CurFn);
1817 }
1818 auto I = FunctionUDMMap.find(CGF.CurFn);
1819 if (I != FunctionUDMMap.end()) {
1820 for(const auto *D : I->second)
1821 UDMMap.erase(D);
1822 FunctionUDMMap.erase(I);
1823 }
1824 LastprivateConditionalToTypes.erase(CGF.CurFn);
1825}
1826
1827llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1828 return IdentTy->getPointerTo();
1829}
1830
1831llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1832 if (!Kmpc_MicroTy) {
1833 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1834 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1835 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1836 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1837 }
1838 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1839}
1840
1841llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1842 llvm::FunctionCallee RTLFn = nullptr;
1843 switch (static_cast<OpenMPRTLFunction>(Function)) {
1844 case OMPRTL__kmpc_fork_call: {
1845 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1846 // microtask, ...);
1847 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1848 getKmpc_MicroPointerTy()};
1849 auto *FnTy =
1850 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1851 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1852 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1853 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1854 llvm::LLVMContext &Ctx = F->getContext();
1855 llvm::MDBuilder MDB(Ctx);
1856 // Annotate the callback behavior of the __kmpc_fork_call:
1857 // - The callback callee is argument number 2 (microtask).
1858 // - The first two arguments of the callback callee are unknown (-1).
1859 // - All variadic arguments to the __kmpc_fork_call are passed to the
1860 // callback callee.
1861 F->addMetadata(
1862 llvm::LLVMContext::MD_callback,
1863 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1864 2, {-1, -1},
1865 /* VarArgsArePassed */ true)}));
1866 }
1867 }
1868 break;
1869 }
1870 case OMPRTL__kmpc_global_thread_num: {
1871 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1872 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1873 auto *FnTy =
1874 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1875 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1876 break;
1877 }
1878 case OMPRTL__kmpc_threadprivate_cached: {
1879 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1880 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1881 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1882 CGM.VoidPtrTy, CGM.SizeTy,
1883 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1884 auto *FnTy =
1885 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1886 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1887 break;
1888 }
1889 case OMPRTL__kmpc_critical: {
1890 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1891 // kmp_critical_name *crit);
1892 llvm::Type *TypeParams[] = {
1893 getIdentTyPointerTy(), CGM.Int32Ty,
1894 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1895 auto *FnTy =
1896 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1897 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1898 break;
1899 }
1900 case OMPRTL__kmpc_critical_with_hint: {
1901 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1902 // kmp_critical_name *crit, uintptr_t hint);
1903 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1904 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1905 CGM.IntPtrTy};
1906 auto *FnTy =
1907 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1908 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1909 break;
1910 }
1911 case OMPRTL__kmpc_threadprivate_register: {
1912 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1913 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1914 // typedef void *(*kmpc_ctor)(void *);
1915 auto *KmpcCtorTy =
1916 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1917 /*isVarArg*/ false)->getPointerTo();
1918 // typedef void *(*kmpc_cctor)(void *, void *);
1919 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1920 auto *KmpcCopyCtorTy =
1921 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1922 /*isVarArg*/ false)
1923 ->getPointerTo();
1924 // typedef void (*kmpc_dtor)(void *);
1925 auto *KmpcDtorTy =
1926 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1927 ->getPointerTo();
1928 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1929 KmpcCopyCtorTy, KmpcDtorTy};
1930 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1931 /*isVarArg*/ false);
1932 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1933 break;
1934 }
1935 case OMPRTL__kmpc_end_critical: {
1936 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1937 // kmp_critical_name *crit);
1938 llvm::Type *TypeParams[] = {
1939 getIdentTyPointerTy(), CGM.Int32Ty,
1940 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1941 auto *FnTy =
1942 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1943 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1944 break;
1945 }
1946 case OMPRTL__kmpc_cancel_barrier: {
1947 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1948 // global_tid);
1949 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1950 auto *FnTy =
1951 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1952 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1953 break;
1954 }
1955 case OMPRTL__kmpc_barrier: {
1956 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1957 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1958 auto *FnTy =
1959 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1960 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1961 break;
1962 }
1963 case OMPRTL__kmpc_for_static_fini: {
1964 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1965 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1966 auto *FnTy =
1967 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1968 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1969 break;
1970 }
1971 case OMPRTL__kmpc_push_num_threads: {
1972 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1973 // kmp_int32 num_threads)
1974 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1975 CGM.Int32Ty};
1976 auto *FnTy =
1977 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1978 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1979 break;
1980 }
1981 case OMPRTL__kmpc_serialized_parallel: {
1982 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1983 // global_tid);
1984 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1985 auto *FnTy =
1986 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1987 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1988 break;
1989 }
1990 case OMPRTL__kmpc_end_serialized_parallel: {
1991 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1992 // global_tid);
1993 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1994 auto *FnTy =
1995 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1996 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1997 break;
1998 }
1999 case OMPRTL__kmpc_flush: {
2000 // Build void __kmpc_flush(ident_t *loc);
2001 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
2002 auto *FnTy =
2003 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2004 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
2005 break;
2006 }
2007 case OMPRTL__kmpc_master: {
2008 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
2009 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2010 auto *FnTy =
2011 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2012 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
2013 break;
2014 }
2015 case OMPRTL__kmpc_end_master: {
2016 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
2017 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2018 auto *FnTy =
2019 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2020 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
2021 break;
2022 }
2023 case OMPRTL__kmpc_omp_taskyield: {
2024 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
2025 // int end_part);
2026 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2027 auto *FnTy =
2028 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2029 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
2030 break;
2031 }
2032 case OMPRTL__kmpc_single: {
2033 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
2034 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2035 auto *FnTy =
2036 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2037 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
2038 break;
2039 }
2040 case OMPRTL__kmpc_end_single: {
2041 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
2042 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2043 auto *FnTy =
2044 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2045 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
2046 break;
2047 }
2048 case OMPRTL__kmpc_omp_task_alloc: {
2049 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
2050 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2051 // kmp_routine_entry_t *task_entry);
2052 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2053, __PRETTY_FUNCTION__))
2053 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2053, __PRETTY_FUNCTION__))
;
2054 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2055 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
2056 // Return void * and then cast to particular kmp_task_t type.
2057 auto *FnTy =
2058 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2059 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
2060 break;
2061 }
2062 case OMPRTL__kmpc_omp_target_task_alloc: {
2063 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
2064 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2065 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
2066 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2067, __PRETTY_FUNCTION__))
2067 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2067, __PRETTY_FUNCTION__))
;
2068 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2069 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
2070 CGM.Int64Ty};
2071 // Return void * and then cast to particular kmp_task_t type.
2072 auto *FnTy =
2073 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2074 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2075 break;
2076 }
2077 case OMPRTL__kmpc_omp_task: {
2078 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2079 // *new_task);
2080 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2081 CGM.VoidPtrTy};
2082 auto *FnTy =
2083 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2084 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2085 break;
2086 }
2087 case OMPRTL__kmpc_copyprivate: {
2088 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2089 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2090 // kmp_int32 didit);
2091 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2092 auto *CpyFnTy =
2093 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2094 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2095 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2096 CGM.Int32Ty};
2097 auto *FnTy =
2098 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2099 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2100 break;
2101 }
2102 case OMPRTL__kmpc_reduce: {
2103 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2104 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2105 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2106 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2107 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2108 /*isVarArg=*/false);
2109 llvm::Type *TypeParams[] = {
2110 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2111 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2112 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2113 auto *FnTy =
2114 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2115 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2116 break;
2117 }
2118 case OMPRTL__kmpc_reduce_nowait: {
2119 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2120 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2121 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2122 // *lck);
2123 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2124 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2125 /*isVarArg=*/false);
2126 llvm::Type *TypeParams[] = {
2127 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2128 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2129 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2130 auto *FnTy =
2131 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2132 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2133 break;
2134 }
2135 case OMPRTL__kmpc_end_reduce: {
2136 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2137 // kmp_critical_name *lck);
2138 llvm::Type *TypeParams[] = {
2139 getIdentTyPointerTy(), CGM.Int32Ty,
2140 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2141 auto *FnTy =
2142 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2143 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2144 break;
2145 }
2146 case OMPRTL__kmpc_end_reduce_nowait: {
2147 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2148 // kmp_critical_name *lck);
2149 llvm::Type *TypeParams[] = {
2150 getIdentTyPointerTy(), CGM.Int32Ty,
2151 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2152 auto *FnTy =
2153 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2154 RTLFn =
2155 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2156 break;
2157 }
2158 case OMPRTL__kmpc_omp_task_begin_if0: {
2159 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2160 // *new_task);
2161 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2162 CGM.VoidPtrTy};
2163 auto *FnTy =
2164 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2165 RTLFn =
2166 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2167 break;
2168 }
2169 case OMPRTL__kmpc_omp_task_complete_if0: {
2170 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2171 // *new_task);
2172 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2173 CGM.VoidPtrTy};
2174 auto *FnTy =
2175 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2176 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2177 /*Name=*/"__kmpc_omp_task_complete_if0");
2178 break;
2179 }
2180 case OMPRTL__kmpc_ordered: {
2181 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2182 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2183 auto *FnTy =
2184 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2185 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2186 break;
2187 }
2188 case OMPRTL__kmpc_end_ordered: {
2189 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2190 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2191 auto *FnTy =
2192 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2193 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2194 break;
2195 }
2196 case OMPRTL__kmpc_omp_taskwait: {
2197 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2198 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2199 auto *FnTy =
2200 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2201 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2202 break;
2203 }
2204 case OMPRTL__kmpc_taskgroup: {
2205 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2206 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2207 auto *FnTy =
2208 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2209 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2210 break;
2211 }
2212 case OMPRTL__kmpc_end_taskgroup: {
2213 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2214 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2215 auto *FnTy =
2216 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2217 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2218 break;
2219 }
2220 case OMPRTL__kmpc_push_proc_bind: {
2221 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2222 // int proc_bind)
2223 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2224 auto *FnTy =
2225 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2226 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2227 break;
2228 }
2229 case OMPRTL__kmpc_omp_task_with_deps: {
2230 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2231 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2232 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2233 llvm::Type *TypeParams[] = {
2234 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2235 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2236 auto *FnTy =
2237 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2238 RTLFn =
2239 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2240 break;
2241 }
2242 case OMPRTL__kmpc_omp_wait_deps: {
2243 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2244 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2245 // kmp_depend_info_t *noalias_dep_list);
2246 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2247 CGM.Int32Ty, CGM.VoidPtrTy,
2248 CGM.Int32Ty, CGM.VoidPtrTy};
2249 auto *FnTy =
2250 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2251 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2252 break;
2253 }
2254 case OMPRTL__kmpc_cancellationpoint: {
2255 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2256 // global_tid, kmp_int32 cncl_kind)
2257 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2258 auto *FnTy =
2259 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2260 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2261 break;
2262 }
2263 case OMPRTL__kmpc_cancel: {
2264 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2265 // kmp_int32 cncl_kind)
2266 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2267 auto *FnTy =
2268 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2269 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2270 break;
2271 }
2272 case OMPRTL__kmpc_push_num_teams: {
2273 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2274 // kmp_int32 num_teams, kmp_int32 num_threads)
2275 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2276 CGM.Int32Ty};
2277 auto *FnTy =
2278 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2279 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2280 break;
2281 }
2282 case OMPRTL__kmpc_fork_teams: {
2283 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2284 // microtask, ...);
2285 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2286 getKmpc_MicroPointerTy()};
2287 auto *FnTy =
2288 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2289 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2290 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2291 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2292 llvm::LLVMContext &Ctx = F->getContext();
2293 llvm::MDBuilder MDB(Ctx);
2294 // Annotate the callback behavior of the __kmpc_fork_teams:
2295 // - The callback callee is argument number 2 (microtask).
2296 // - The first two arguments of the callback callee are unknown (-1).
2297 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2298 // callback callee.
2299 F->addMetadata(
2300 llvm::LLVMContext::MD_callback,
2301 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2302 2, {-1, -1},
2303 /* VarArgsArePassed */ true)}));
2304 }
2305 }
2306 break;
2307 }
2308 case OMPRTL__kmpc_taskloop: {
2309 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2310 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2311 // sched, kmp_uint64 grainsize, void *task_dup);
2312 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2313 CGM.IntTy,
2314 CGM.VoidPtrTy,
2315 CGM.IntTy,
2316 CGM.Int64Ty->getPointerTo(),
2317 CGM.Int64Ty->getPointerTo(),
2318 CGM.Int64Ty,
2319 CGM.IntTy,
2320 CGM.IntTy,
2321 CGM.Int64Ty,
2322 CGM.VoidPtrTy};
2323 auto *FnTy =
2324 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2325 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2326 break;
2327 }
2328 case OMPRTL__kmpc_doacross_init: {
2329 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2330 // num_dims, struct kmp_dim *dims);
2331 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2332 CGM.Int32Ty,
2333 CGM.Int32Ty,
2334 CGM.VoidPtrTy};
2335 auto *FnTy =
2336 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2337 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2338 break;
2339 }
2340 case OMPRTL__kmpc_doacross_fini: {
2341 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2342 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2343 auto *FnTy =
2344 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2345 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2346 break;
2347 }
2348 case OMPRTL__kmpc_doacross_post: {
2349 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2350 // *vec);
2351 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2352 CGM.Int64Ty->getPointerTo()};
2353 auto *FnTy =
2354 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2355 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2356 break;
2357 }
2358 case OMPRTL__kmpc_doacross_wait: {
2359 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2360 // *vec);
2361 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2362 CGM.Int64Ty->getPointerTo()};
2363 auto *FnTy =
2364 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2365 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2366 break;
2367 }
2368 case OMPRTL__kmpc_task_reduction_init: {
2369 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2370 // *data);
2371 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2372 auto *FnTy =
2373 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2374 RTLFn =
2375 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2376 break;
2377 }
2378 case OMPRTL__kmpc_task_reduction_get_th_data: {
2379 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2380 // *d);
2381 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2382 auto *FnTy =
2383 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2384 RTLFn = CGM.CreateRuntimeFunction(
2385 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2386 break;
2387 }
2388 case OMPRTL__kmpc_alloc: {
2389 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2390 // al); omp_allocator_handle_t type is void *.
2391 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2392 auto *FnTy =
2393 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2394 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2395 break;
2396 }
2397 case OMPRTL__kmpc_free: {
2398 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2399 // al); omp_allocator_handle_t type is void *.
2400 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2401 auto *FnTy =
2402 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2403 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2404 break;
2405 }
2406 case OMPRTL__kmpc_push_target_tripcount: {
2407 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2408 // size);
2409 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2410 llvm::FunctionType *FnTy =
2411 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2412 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2413 break;
2414 }
2415 case OMPRTL__tgt_target: {
2416 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2417 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2418 // *arg_types);
2419 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2420 CGM.VoidPtrTy,
2421 CGM.Int32Ty,
2422 CGM.VoidPtrPtrTy,
2423 CGM.VoidPtrPtrTy,
2424 CGM.Int64Ty->getPointerTo(),
2425 CGM.Int64Ty->getPointerTo()};
2426 auto *FnTy =
2427 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2428 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2429 break;
2430 }
2431 case OMPRTL__tgt_target_nowait: {
2432 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2433 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2434 // int64_t *arg_types);
2435 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2436 CGM.VoidPtrTy,
2437 CGM.Int32Ty,
2438 CGM.VoidPtrPtrTy,
2439 CGM.VoidPtrPtrTy,
2440 CGM.Int64Ty->getPointerTo(),
2441 CGM.Int64Ty->getPointerTo()};
2442 auto *FnTy =
2443 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2444 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2445 break;
2446 }
2447 case OMPRTL__tgt_target_teams: {
2448 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2449 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2450 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2451 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2452 CGM.VoidPtrTy,
2453 CGM.Int32Ty,
2454 CGM.VoidPtrPtrTy,
2455 CGM.VoidPtrPtrTy,
2456 CGM.Int64Ty->getPointerTo(),
2457 CGM.Int64Ty->getPointerTo(),
2458 CGM.Int32Ty,
2459 CGM.Int32Ty};
2460 auto *FnTy =
2461 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2462 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2463 break;
2464 }
2465 case OMPRTL__tgt_target_teams_nowait: {
2466 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2467 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2468 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2469 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2470 CGM.VoidPtrTy,
2471 CGM.Int32Ty,
2472 CGM.VoidPtrPtrTy,
2473 CGM.VoidPtrPtrTy,
2474 CGM.Int64Ty->getPointerTo(),
2475 CGM.Int64Ty->getPointerTo(),
2476 CGM.Int32Ty,
2477 CGM.Int32Ty};
2478 auto *FnTy =
2479 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2480 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2481 break;
2482 }
2483 case OMPRTL__tgt_register_requires: {
2484 // Build void __tgt_register_requires(int64_t flags);
2485 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2486 auto *FnTy =
2487 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2488 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2489 break;
2490 }
2491 case OMPRTL__tgt_target_data_begin: {
2492 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2493 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2494 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2495 CGM.Int32Ty,
2496 CGM.VoidPtrPtrTy,
2497 CGM.VoidPtrPtrTy,
2498 CGM.Int64Ty->getPointerTo(),
2499 CGM.Int64Ty->getPointerTo()};
2500 auto *FnTy =
2501 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2502 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2503 break;
2504 }
2505 case OMPRTL__tgt_target_data_begin_nowait: {
2506 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2507 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2508 // *arg_types);
2509 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2510 CGM.Int32Ty,
2511 CGM.VoidPtrPtrTy,
2512 CGM.VoidPtrPtrTy,
2513 CGM.Int64Ty->getPointerTo(),
2514 CGM.Int64Ty->getPointerTo()};
2515 auto *FnTy =
2516 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2517 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2518 break;
2519 }
2520 case OMPRTL__tgt_target_data_end: {
2521 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2522 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2523 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2524 CGM.Int32Ty,
2525 CGM.VoidPtrPtrTy,
2526 CGM.VoidPtrPtrTy,
2527 CGM.Int64Ty->getPointerTo(),
2528 CGM.Int64Ty->getPointerTo()};
2529 auto *FnTy =
2530 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2531 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2532 break;
2533 }
2534 case OMPRTL__tgt_target_data_end_nowait: {
2535 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2536 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2537 // *arg_types);
2538 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2539 CGM.Int32Ty,
2540 CGM.VoidPtrPtrTy,
2541 CGM.VoidPtrPtrTy,
2542 CGM.Int64Ty->getPointerTo(),
2543 CGM.Int64Ty->getPointerTo()};
2544 auto *FnTy =
2545 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2546 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2547 break;
2548 }
2549 case OMPRTL__tgt_target_data_update: {
2550 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2551 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2552 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2553 CGM.Int32Ty,
2554 CGM.VoidPtrPtrTy,
2555 CGM.VoidPtrPtrTy,
2556 CGM.Int64Ty->getPointerTo(),
2557 CGM.Int64Ty->getPointerTo()};
2558 auto *FnTy =
2559 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2560 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2561 break;
2562 }
2563 case OMPRTL__tgt_target_data_update_nowait: {
2564 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2565 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2566 // *arg_types);
2567 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2568 CGM.Int32Ty,
2569 CGM.VoidPtrPtrTy,
2570 CGM.VoidPtrPtrTy,
2571 CGM.Int64Ty->getPointerTo(),
2572 CGM.Int64Ty->getPointerTo()};
2573 auto *FnTy =
2574 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2575 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2576 break;
2577 }
2578 case OMPRTL__tgt_mapper_num_components: {
2579 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2580 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2581 auto *FnTy =
2582 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2583 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2584 break;
2585 }
2586 case OMPRTL__tgt_push_mapper_component: {
2587 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2588 // *base, void *begin, int64_t size, int64_t type);
2589 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2590 CGM.Int64Ty, CGM.Int64Ty};
2591 auto *FnTy =
2592 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2593 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2594 break;
2595 }
2596 }
2597 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2597, __PRETTY_FUNCTION__))
;
2598 return RTLFn;
2599}
2600
2601llvm::FunctionCallee
2602CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2603 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2604, __PRETTY_FUNCTION__))
2604 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2604, __PRETTY_FUNCTION__))
;
2605 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2606 : "__kmpc_for_static_init_4u")
2607 : (IVSigned ? "__kmpc_for_static_init_8"
2608 : "__kmpc_for_static_init_8u");
2609 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2610 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2611 llvm::Type *TypeParams[] = {
2612 getIdentTyPointerTy(), // loc
2613 CGM.Int32Ty, // tid
2614 CGM.Int32Ty, // schedtype
2615 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2616 PtrTy, // p_lower
2617 PtrTy, // p_upper
2618 PtrTy, // p_stride
2619 ITy, // incr
2620 ITy // chunk
2621 };
2622 auto *FnTy =
2623 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2624 return CGM.CreateRuntimeFunction(FnTy, Name);
2625}
2626
2627llvm::FunctionCallee
2628CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2629 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2630, __PRETTY_FUNCTION__))
2630 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2630, __PRETTY_FUNCTION__))
;
2631 StringRef Name =
2632 IVSize == 32
2633 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2634 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2635 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2636 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2637 CGM.Int32Ty, // tid
2638 CGM.Int32Ty, // schedtype
2639 ITy, // lower
2640 ITy, // upper
2641 ITy, // stride
2642 ITy // chunk
2643 };
2644 auto *FnTy =
2645 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2646 return CGM.CreateRuntimeFunction(FnTy, Name);
2647}
2648
2649llvm::FunctionCallee
2650CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2651 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2652, __PRETTY_FUNCTION__))
2652 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2652, __PRETTY_FUNCTION__))
;
2653 StringRef Name =
2654 IVSize == 32
2655 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2656 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2657 llvm::Type *TypeParams[] = {
2658 getIdentTyPointerTy(), // loc
2659 CGM.Int32Ty, // tid
2660 };
2661 auto *FnTy =
2662 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2663 return CGM.CreateRuntimeFunction(FnTy, Name);
2664}
2665
2666llvm::FunctionCallee
2667CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2668 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2669, __PRETTY_FUNCTION__))
2669 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2669, __PRETTY_FUNCTION__))
;
2670 StringRef Name =
2671 IVSize == 32
2672 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2673 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2674 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2675 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2676 llvm::Type *TypeParams[] = {
2677 getIdentTyPointerTy(), // loc
2678 CGM.Int32Ty, // tid
2679 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2680 PtrTy, // p_lower
2681 PtrTy, // p_upper
2682 PtrTy // p_stride
2683 };
2684 auto *FnTy =
2685 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2686 return CGM.CreateRuntimeFunction(FnTy, Name);
2687}
2688
2689/// Obtain information that uniquely identifies a target entry. This
2690/// consists of the file and device IDs as well as line number associated with
2691/// the relevant entry source location.
2692static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2693 unsigned &DeviceID, unsigned &FileID,
2694 unsigned &LineNum) {
2695 SourceManager &SM = C.getSourceManager();
2696
2697 // The loc should be always valid and have a file ID (the user cannot use
2698 // #pragma directives in macros)
2699
2700 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2700, __PRETTY_FUNCTION__))
;
2701
2702 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2703 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2703, __PRETTY_FUNCTION__))
;
2704
2705 llvm::sys::fs::UniqueID ID;
2706 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2707 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2708 << PLoc.getFilename() << EC.message();
2709
2710 DeviceID = ID.getDevice();
2711 FileID = ID.getFile();
2712 LineNum = PLoc.getLine();
2713}
2714
2715Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2716 if (CGM.getLangOpts().OpenMPSimd)
2717 return Address::invalid();
2718 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2719 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2720 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2721 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2722 HasRequiresUnifiedSharedMemory))) {
2723 SmallString<64> PtrName;
2724 {
2725 llvm::raw_svector_ostream OS(PtrName);
2726 OS << CGM.getMangledName(GlobalDecl(VD));
2727 if (!VD->isExternallyVisible()) {
2728 unsigned DeviceID, FileID, Line;
2729 getTargetEntryUniqueInfo(CGM.getContext(),
2730 VD->getCanonicalDecl()->getBeginLoc(),
2731 DeviceID, FileID, Line);
2732 OS << llvm::format("_%x", FileID);
2733 }
2734 OS << "_decl_tgt_ref_ptr";
2735 }
2736 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2737 if (!Ptr) {
2738 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2739 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2740 PtrName);
2741
2742 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2743 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2744
2745 if (!CGM.getLangOpts().OpenMPIsDevice)
2746 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2747 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2748 }
2749 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2750 }
2751 return Address::invalid();
2752}
2753
2754llvm::Constant *
2755CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2756 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2757, __PRETTY_FUNCTION__))
2757 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2757, __PRETTY_FUNCTION__))
;
2758 // Lookup the entry, lazily creating it if necessary.
2759 std::string Suffix = getName({"cache", ""});
2760 return getOrCreateInternalVariable(
2761 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2762}
2763
2764Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2765 const VarDecl *VD,
2766 Address VDAddr,
2767 SourceLocation Loc) {
2768 if (CGM.getLangOpts().OpenMPUseTLS &&
2769 CGM.getContext().getTargetInfo().isTLSSupported())
2770 return VDAddr;
2771
2772 llvm::Type *VarTy = VDAddr.getElementType();
2773 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2774 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2775 CGM.Int8PtrTy),
2776 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2777 getOrCreateThreadPrivateCache(VD)};
2778 return Address(CGF.EmitRuntimeCall(
2779 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2780 VDAddr.getAlignment());
2781}
2782
2783void CGOpenMPRuntime::emitThreadPrivateVarInit(
2784 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2785 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2786 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2787 // library.
2788 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2789 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2790 OMPLoc);
2791 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2792 // to register constructor/destructor for variable.
2793 llvm::Value *Args[] = {
2794 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2795 Ctor, CopyCtor, Dtor};
2796 CGF.EmitRuntimeCall(
2797 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2798}
2799
2800llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2801 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2802 bool PerformInit, CodeGenFunction *CGF) {
2803 if (CGM.getLangOpts().OpenMPUseTLS &&
2804 CGM.getContext().getTargetInfo().isTLSSupported())
2805 return nullptr;
2806
2807 VD = VD->getDefinition(CGM.getContext());
2808 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2809 QualType ASTTy = VD->getType();
2810
2811 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2812 const Expr *Init = VD->getAnyInitializer();
2813 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2814 // Generate function that re-emits the declaration's initializer into the
2815 // threadprivate copy of the variable VD
2816 CodeGenFunction CtorCGF(CGM);
2817 FunctionArgList Args;
2818 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2819 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2820 ImplicitParamDecl::Other);
2821 Args.push_back(&Dst);
2822
2823 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2824 CGM.getContext().VoidPtrTy, Args);
2825 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2826 std::string Name = getName({"__kmpc_global_ctor_", ""});
2827 llvm::Function *Fn =
2828 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2829 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2830 Args, Loc, Loc);
2831 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2832 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2833 CGM.getContext().VoidPtrTy, Dst.getLocation());
2834 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2835 Arg = CtorCGF.Builder.CreateElementBitCast(
2836 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2837 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2838 /*IsInitializer=*/true);
2839 ArgVal = CtorCGF.EmitLoadOfScalar(
2840 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2841 CGM.getContext().VoidPtrTy, Dst.getLocation());
2842 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2843 CtorCGF.FinishFunction();
2844 Ctor = Fn;
2845 }
2846 if (VD->getType().isDestructedType() != QualType::DK_none) {
2847 // Generate function that emits destructor call for the threadprivate copy
2848 // of the variable VD
2849 CodeGenFunction DtorCGF(CGM);
2850 FunctionArgList Args;
2851 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2852 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2853 ImplicitParamDecl::Other);
2854 Args.push_back(&Dst);
2855
2856 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2857 CGM.getContext().VoidTy, Args);
2858 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2859 std::string Name = getName({"__kmpc_global_dtor_", ""});
2860 llvm::Function *Fn =
2861 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2862 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2863 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2864 Loc, Loc);
2865 // Create a scope with an artificial location for the body of this function.
2866 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2867 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2868 DtorCGF.GetAddrOfLocalVar(&Dst),
2869 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2870 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2871 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2872 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2873 DtorCGF.FinishFunction();
2874 Dtor = Fn;
2875 }
2876 // Do not emit init function if it is not required.
2877 if (!Ctor && !Dtor)
2878 return nullptr;
2879
2880 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2881 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2882 /*isVarArg=*/false)
2883 ->getPointerTo();
2884 // Copying constructor for the threadprivate variable.
2885 // Must be NULL - reserved by runtime, but currently it requires that this
2886 // parameter is always NULL. Otherwise it fires assertion.
2887 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2888 if (Ctor == nullptr) {
2889 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2890 /*isVarArg=*/false)
2891 ->getPointerTo();
2892 Ctor = llvm::Constant::getNullValue(CtorTy);
2893 }
2894 if (Dtor == nullptr) {
2895 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2896 /*isVarArg=*/false)
2897 ->getPointerTo();
2898 Dtor = llvm::Constant::getNullValue(DtorTy);
2899 }
2900 if (!CGF) {
2901 auto *InitFunctionTy =
2902 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2903 std::string Name = getName({"__omp_threadprivate_init_", ""});
2904 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2905 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2906 CodeGenFunction InitCGF(CGM);
2907 FunctionArgList ArgList;
2908 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2909 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2910 Loc, Loc);
2911 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2912 InitCGF.FinishFunction();
2913 return InitFunction;
2914 }
2915 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2916 }
2917 return nullptr;
2918}
2919
2920bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2921 llvm::GlobalVariable *Addr,
2922 bool PerformInit) {
2923 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2924 !CGM.getLangOpts().OpenMPIsDevice)
2925 return false;
2926 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2927 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2928 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2929 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2930 HasRequiresUnifiedSharedMemory))
2931 return CGM.getLangOpts().OpenMPIsDevice;
2932 VD = VD->getDefinition(CGM.getContext());
2933 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2934 return CGM.getLangOpts().OpenMPIsDevice;
2935
2936 QualType ASTTy = VD->getType();
2937
2938 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2939 // Produce the unique prefix to identify the new target regions. We use
2940 // the source location of the variable declaration which we know to not
2941 // conflict with any target region.
2942 unsigned DeviceID;
2943 unsigned FileID;
2944 unsigned Line;
2945 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2946 SmallString<128> Buffer, Out;
2947 {
2948 llvm::raw_svector_ostream OS(Buffer);
2949 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2950 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2951 }
2952
2953 const Expr *Init = VD->getAnyInitializer();
2954 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2955 llvm::Constant *Ctor;
2956 llvm::Constant *ID;
2957 if (CGM.getLangOpts().OpenMPIsDevice) {
2958 // Generate function that re-emits the declaration's initializer into
2959 // the threadprivate copy of the variable VD
2960 CodeGenFunction CtorCGF(CGM);
2961
2962 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2963 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2964 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2965 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2966 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2967 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2968 FunctionArgList(), Loc, Loc);
2969 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2970 CtorCGF.EmitAnyExprToMem(Init,
2971 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2972 Init->getType().getQualifiers(),
2973 /*IsInitializer=*/true);
2974 CtorCGF.FinishFunction();
2975 Ctor = Fn;
2976 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2977 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2978 } else {
2979 Ctor = new llvm::GlobalVariable(
2980 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2981 llvm::GlobalValue::PrivateLinkage,
2982 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2983 ID = Ctor;
2984 }
2985
2986 // Register the information for the entry associated with the constructor.
2987 Out.clear();
2988 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2989 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2990 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2991 }
2992 if (VD->getType().isDestructedType() != QualType::DK_none) {
2993 llvm::Constant *Dtor;
2994 llvm::Constant *ID;
2995 if (CGM.getLangOpts().OpenMPIsDevice) {
2996 // Generate function that emits destructor call for the threadprivate
2997 // copy of the variable VD
2998 CodeGenFunction DtorCGF(CGM);
2999
3000 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3001 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3002 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
3003 FTy, Twine(Buffer, "_dtor"), FI, Loc);
3004 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
3005 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
3006 FunctionArgList(), Loc, Loc);
3007 // Create a scope with an artificial location for the body of this
3008 // function.
3009 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
3010 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
3011 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
3012 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
3013 DtorCGF.FinishFunction();
3014 Dtor = Fn;
3015 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
3016 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
3017 } else {
3018 Dtor = new llvm::GlobalVariable(
3019 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
3020 llvm::GlobalValue::PrivateLinkage,
3021 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
3022 ID = Dtor;
3023 }
3024 // Register the information for the entry associated with the destructor.
3025 Out.clear();
3026 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
3027 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
3028 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
3029 }
3030 return CGM.getLangOpts().OpenMPIsDevice;
3031}
3032
3033Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
3034 QualType VarType,
3035 StringRef Name) {
3036 std::string Suffix = getName({"artificial", ""});
3037 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
3038 llvm::Value *GAddr =
3039 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
3040 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
3041 CGM.getTarget().isTLSSupported()) {
3042 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
3043 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
3044 }
3045 std::string CacheSuffix = getName({"cache", ""});
3046 llvm::Value *Args[] = {
3047 emitUpdateLocation(CGF, SourceLocation()),
3048 getThreadID(CGF, SourceLocation()),
3049 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
3050 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
3051 /*isSigned=*/false),
3052 getOrCreateInternalVariable(
3053 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
3054 return Address(
3055 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3056 CGF.EmitRuntimeCall(
3057 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3058 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3059 CGM.getContext().getTypeAlignInChars(VarType));
3060}
3061
3062void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
3063 const RegionCodeGenTy &ThenGen,
3064 const RegionCodeGenTy &ElseGen) {
3065 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3066
3067 // If the condition constant folds and can be elided, try to avoid emitting
3068 // the condition and the dead arm of the if/else.
3069 bool CondConstant;
3070 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
9
Assuming the condition is false
10
Taking false branch
3071 if (CondConstant)
3072 ThenGen(CGF);
3073 else
3074 ElseGen(CGF);
3075 return;
3076 }
3077
3078 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3079 // emit the conditional branch.
3080 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3081 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3082 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3083 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3084
3085 // Emit the 'then' code.
3086 CGF.EmitBlock(ThenBlock);
3087 ThenGen(CGF);
11
Calling 'RegionCodeGenTy::operator()'
3088 CGF.EmitBranch(ContBlock);
3089 // Emit the 'else' code if present.
3090 // There is no need to emit line number for unconditional branch.
3091 (void)ApplyDebugLocation::CreateEmpty(CGF);
3092 CGF.EmitBlock(ElseBlock);
3093 ElseGen(CGF);
3094 // There is no need to emit line number for unconditional branch.
3095 (void)ApplyDebugLocation::CreateEmpty(CGF);
3096 CGF.EmitBranch(ContBlock);
3097 // Emit the continuation block for code after the if.
3098 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3099}
3100
3101void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3102 llvm::Function *OutlinedFn,
3103 ArrayRef<llvm::Value *> CapturedVars,
3104 const Expr *IfCond) {
3105 if (!CGF.HaveInsertPoint())
3106 return;
3107 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3108 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3109 PrePostActionTy &) {
3110 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3111 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3112 llvm::Value *Args[] = {
3113 RTLoc,
3114 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3115 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3116 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3117 RealArgs.append(std::begin(Args), std::end(Args));
3118 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3119
3120 llvm::FunctionCallee RTLFn =
3121 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3122 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3123 };
3124 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3125 PrePostActionTy &) {
3126 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3127 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3128 // Build calls:
3129 // __kmpc_serialized_parallel(&Loc, GTid);
3130 llvm::Value *Args[] = {RTLoc, ThreadID};
3131 CGF.EmitRuntimeCall(
3132 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3133
3134 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
3135 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
3136 Address ZeroAddrBound =
3137 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3138 /*Name=*/".bound.zero.addr");
3139 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
3140 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3141 // ThreadId for serialized parallels is 0.
3142 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
3143 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
3144 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3145 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3146
3147 // __kmpc_end_serialized_parallel(&Loc, GTid);
3148 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3149 CGF.EmitRuntimeCall(
3150 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3151 EndArgs);
3152 };
3153 if (IfCond) {
3154 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3155 } else {
3156 RegionCodeGenTy ThenRCG(ThenGen);
3157 ThenRCG(CGF);
3158 }
3159}
3160
3161// If we're inside an (outlined) parallel region, use the region info's
3162// thread-ID variable (it is passed in a first argument of the outlined function
3163// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3164// regular serial code region, get thread ID by calling kmp_int32
3165// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3166// return the address of that temp.
3167Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3168 SourceLocation Loc) {
3169 if (auto *OMPRegionInfo =
3170 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3171 if (OMPRegionInfo->getThreadIDVariable())
3172 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3173
3174 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3175 QualType Int32Ty =
3176 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3177 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3178 CGF.EmitStoreOfScalar(ThreadID,
3179 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3180
3181 return ThreadIDTemp;
3182}
3183
3184llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3185 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3186 SmallString<256> Buffer;
3187 llvm::raw_svector_ostream Out(Buffer);
3188 Out << Name;
3189 StringRef RuntimeName = Out.str();
3190 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3191 if (Elem.second) {
3192 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3193, __PRETTY_FUNCTION__))
3193 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3193, __PRETTY_FUNCTION__))
;
3194 return &*Elem.second;
3195 }
3196
3197 return Elem.second = new llvm::GlobalVariable(
3198 CGM.getModule(), Ty, /*IsConstant*/ false,
3199 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3200 Elem.first(), /*InsertBefore=*/nullptr,
3201 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3202}
3203
3204llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3205 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3206 std::string Name = getName({Prefix, "var"});
3207 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3208}
3209
3210namespace {
3211/// Common pre(post)-action for different OpenMP constructs.
3212class CommonActionTy final : public PrePostActionTy {
3213 llvm::FunctionCallee EnterCallee;
3214 ArrayRef<llvm::Value *> EnterArgs;
3215 llvm::FunctionCallee ExitCallee;
3216 ArrayRef<llvm::Value *> ExitArgs;
3217 bool Conditional;
3218 llvm::BasicBlock *ContBlock = nullptr;
3219
3220public:
3221 CommonActionTy(llvm::FunctionCallee EnterCallee,
3222 ArrayRef<llvm::Value *> EnterArgs,
3223 llvm::FunctionCallee ExitCallee,
3224 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3225 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3226 ExitArgs(ExitArgs), Conditional(Conditional) {}
3227 void Enter(CodeGenFunction &CGF) override {
3228 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3229 if (Conditional) {
3230 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3231 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3232 ContBlock = CGF.createBasicBlock("omp_if.end");
3233 // Generate the branch (If-stmt)
3234 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3235 CGF.EmitBlock(ThenBlock);
3236 }
3237 }
3238 void Done(CodeGenFunction &CGF) {
3239 // Emit the rest of blocks/branches
3240 CGF.EmitBranch(ContBlock);
3241 CGF.EmitBlock(ContBlock, true);
3242 }
3243 void Exit(CodeGenFunction &CGF) override {
3244 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3245 }
3246};
3247} // anonymous namespace
3248
3249void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3250 StringRef CriticalName,
3251 const RegionCodeGenTy &CriticalOpGen,
3252 SourceLocation Loc, const Expr *Hint) {
3253 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3254 // CriticalOpGen();
3255 // __kmpc_end_critical(ident_t *, gtid, Lock);
3256 // Prepare arguments and build a call to __kmpc_critical
3257 if (!CGF.HaveInsertPoint())
3258 return;
3259 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3260 getCriticalRegionLock(CriticalName)};
3261 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3262 std::end(Args));
3263 if (Hint) {
3264 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3265 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3266 }
3267 CommonActionTy Action(
3268 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3269 : OMPRTL__kmpc_critical),
3270 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3271 CriticalOpGen.setAction(Action);
3272 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3273}
3274
3275void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3276 const RegionCodeGenTy &MasterOpGen,
3277 SourceLocation Loc) {
3278 if (!CGF.HaveInsertPoint())
3279 return;
3280 // if(__kmpc_master(ident_t *, gtid)) {
3281 // MasterOpGen();
3282 // __kmpc_end_master(ident_t *, gtid);
3283 // }
3284 // Prepare arguments and build a call to __kmpc_master
3285 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3286 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3287 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3288 /*Conditional=*/true);
3289 MasterOpGen.setAction(Action);
3290 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3291 Action.Done(CGF);
3292}
3293
3294void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3295 SourceLocation Loc) {
3296 if (!CGF.HaveInsertPoint())
3297 return;
3298 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3299 if (OMPBuilder) {
3300 OMPBuilder->CreateTaskyield(CGF.Builder);
3301 } else {
3302 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3303 llvm::Value *Args[] = {
3304 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3305 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3306 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield),
3307 Args);
3308 }
3309
3310 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3311 Region->emitUntiedSwitch(CGF);
3312}
3313
3314void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3315 const RegionCodeGenTy &TaskgroupOpGen,
3316 SourceLocation Loc) {
3317 if (!CGF.HaveInsertPoint())
3318 return;
3319 // __kmpc_taskgroup(ident_t *, gtid);
3320 // TaskgroupOpGen();
3321 // __kmpc_end_taskgroup(ident_t *, gtid);
3322 // Prepare arguments and build a call to __kmpc_taskgroup
3323 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3324 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3325 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3326 Args);
3327 TaskgroupOpGen.setAction(Action);
3328 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3329}
3330
3331/// Given an array of pointers to variables, project the address of a
3332/// given variable.
3333static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3334 unsigned Index, const VarDecl *Var) {
3335 // Pull out the pointer to the variable.
3336 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3337 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3338
3339 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3340 Addr = CGF.Builder.CreateElementBitCast(
3341 Addr, CGF.ConvertTypeForMem(Var->getType()));
3342 return Addr;
3343}
3344
3345static llvm::Value *emitCopyprivateCopyFunction(
3346 CodeGenModule &CGM, llvm::Type *ArgsType,
3347 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3348 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3349 SourceLocation Loc) {
3350 ASTContext &C = CGM.getContext();
3351 // void copy_func(void *LHSArg, void *RHSArg);
3352 FunctionArgList Args;
3353 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3354 ImplicitParamDecl::Other);
3355 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3356 ImplicitParamDecl::Other);
3357 Args.push_back(&LHSArg);
3358 Args.push_back(&RHSArg);
3359 const auto &CGFI =
3360 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3361 std::string Name =
3362 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3363 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3364 llvm::GlobalValue::InternalLinkage, Name,
3365 &CGM.getModule());
3366 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3367 Fn->setDoesNotRecurse();
3368 CodeGenFunction CGF(CGM);
3369 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3370 // Dest = (void*[n])(LHSArg);
3371 // Src = (void*[n])(RHSArg);
3372 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3373 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3374 ArgsType), CGF.getPointerAlign());
3375 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3376 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3377 ArgsType), CGF.getPointerAlign());
3378 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3379 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3380 // ...
3381 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3382 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3383 const auto *DestVar =
3384 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3385 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3386
3387 const auto *SrcVar =
3388 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3389 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3390
3391 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3392 QualType Type = VD->getType();
3393 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3394 }
3395 CGF.FinishFunction();
3396 return Fn;
3397}
3398
3399void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3400 const RegionCodeGenTy &SingleOpGen,
3401 SourceLocation Loc,
3402 ArrayRef<const Expr *> CopyprivateVars,
3403 ArrayRef<const Expr *> SrcExprs,
3404 ArrayRef<const Expr *> DstExprs,
3405 ArrayRef<const Expr *> AssignmentOps) {
3406 if (!CGF.HaveInsertPoint())
3407 return;
3408 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3410, __PRETTY_FUNCTION__))
3409 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3410, __PRETTY_FUNCTION__))
3410 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3410, __PRETTY_FUNCTION__))
;
3411 ASTContext &C = CGM.getContext();
3412 // int32 did_it = 0;
3413 // if(__kmpc_single(ident_t *, gtid)) {
3414 // SingleOpGen();
3415 // __kmpc_end_single(ident_t *, gtid);
3416 // did_it = 1;
3417 // }
3418 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3419 // <copy_func>, did_it);
3420
3421 Address DidIt = Address::invalid();
3422 if (!CopyprivateVars.empty()) {
3423 // int32 did_it = 0;
3424 QualType KmpInt32Ty =
3425 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3426 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3427 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3428 }
3429 // Prepare arguments and build a call to __kmpc_single
3430 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3431 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3432 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3433 /*Conditional=*/true);
3434 SingleOpGen.setAction(Action);
3435 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3436 if (DidIt.isValid()) {
3437 // did_it = 1;
3438 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3439 }
3440 Action.Done(CGF);
3441 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3442 // <copy_func>, did_it);
3443 if (DidIt.isValid()) {
3444 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3445 QualType CopyprivateArrayTy = C.getConstantArrayType(
3446 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3447 /*IndexTypeQuals=*/0);
3448 // Create a list of all private variables for copyprivate.
3449 Address CopyprivateList =
3450 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3451 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3452 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3453 CGF.Builder.CreateStore(
3454 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3455 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
3456 CGF.VoidPtrTy),
3457 Elem);
3458 }
3459 // Build function that copies private values from single region to all other
3460 // threads in the corresponding parallel region.
3461 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3462 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3463 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3464 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3465 Address CL =
3466 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3467 CGF.VoidPtrTy);
3468 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3469 llvm::Value *Args[] = {
3470 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3471 getThreadID(CGF, Loc), // i32 <gtid>
3472 BufSize, // size_t <buf_size>
3473 CL.getPointer(), // void *<copyprivate list>
3474 CpyFn, // void (*) (void *, void *) <copy_func>
3475 DidItVal // i32 did_it
3476 };
3477 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3478 }
3479}
3480
3481void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3482 const RegionCodeGenTy &OrderedOpGen,
3483 SourceLocation Loc, bool IsThreads) {
3484 if (!CGF.HaveInsertPoint())
3485 return;
3486 // __kmpc_ordered(ident_t *, gtid);
3487 // OrderedOpGen();
3488 // __kmpc_end_ordered(ident_t *, gtid);
3489 // Prepare arguments and build a call to __kmpc_ordered
3490 if (IsThreads) {
3491 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3492 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3493 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3494 Args);
3495 OrderedOpGen.setAction(Action);
3496 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3497 return;
3498 }
3499 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3500}
3501
3502unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3503 unsigned Flags;
3504 if (Kind == OMPD_for)
3505 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3506 else if (Kind == OMPD_sections)
3507 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3508 else if (Kind == OMPD_single)
3509 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3510 else if (Kind == OMPD_barrier)
3511 Flags = OMP_IDENT_BARRIER_EXPL;
3512 else
3513 Flags = OMP_IDENT_BARRIER_IMPL;
3514 return Flags;
3515}
3516
3517void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3518 CodeGenFunction &CGF, const OMPLoopDirective &S,
3519 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3520 // Check if the loop directive is actually a doacross loop directive. In this
3521 // case choose static, 1 schedule.
3522 if (llvm::any_of(
3523 S.getClausesOfKind<OMPOrderedClause>(),
3524 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3525 ScheduleKind = OMPC_SCHEDULE_static;
3526 // Chunk size is 1 in this case.
3527 llvm::APInt ChunkSize(32, 1);
3528 ChunkExpr = IntegerLiteral::Create(
3529 CGF.getContext(), ChunkSize,
3530 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3531 SourceLocation());
3532 }
3533}
3534
3535void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3536 OpenMPDirectiveKind Kind, bool EmitChecks,
3537 bool ForceSimpleCall) {
3538 // Check if we should use the OMPBuilder
3539 auto *OMPRegionInfo =
3540 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
3541 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3542 if (OMPBuilder) {
3543 CGF.Builder.restoreIP(OMPBuilder->CreateBarrier(
3544 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
3545 return;
3546 }
3547
3548 if (!CGF.HaveInsertPoint())
3549 return;
3550 // Build call __kmpc_cancel_barrier(loc, thread_id);
3551 // Build call __kmpc_barrier(loc, thread_id);
3552 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3553 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3554 // thread_id);
3555 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3556 getThreadID(CGF, Loc)};
3557 if (OMPRegionInfo) {
3558 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3559 llvm::Value *Result = CGF.EmitRuntimeCall(
3560 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3561 if (EmitChecks) {
3562 // if (__kmpc_cancel_barrier()) {
3563 // exit from construct;
3564 // }
3565 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3566 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3567 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3568 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3569 CGF.EmitBlock(ExitBB);
3570 // exit from construct;
3571 CodeGenFunction::JumpDest CancelDestination =
3572 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3573 CGF.EmitBranchThroughCleanup(CancelDestination);
3574 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3575 }
3576 return;
3577 }
3578 }
3579 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3580}
3581
3582/// Map the OpenMP loop schedule to the runtime enumeration.
3583static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3584 bool Chunked, bool Ordered) {
3585 switch (ScheduleKind) {
3586 case OMPC_SCHEDULE_static:
3587 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3588 : (Ordered ? OMP_ord_static : OMP_sch_static);
3589 case OMPC_SCHEDULE_dynamic:
3590 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3591 case OMPC_SCHEDULE_guided:
3592 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3593 case OMPC_SCHEDULE_runtime:
3594 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3595 case OMPC_SCHEDULE_auto:
3596 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3597 case OMPC_SCHEDULE_unknown:
3598 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3598, __PRETTY_FUNCTION__))
;
3599 return Ordered ? OMP_ord_static : OMP_sch_static;
3600 }
3601 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3601)
;
3602}
3603
3604/// Map the OpenMP distribute schedule to the runtime enumeration.
3605static OpenMPSchedType
3606getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3607 // only static is allowed for dist_schedule
3608 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3609}
3610
3611bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3612 bool Chunked) const {
3613 OpenMPSchedType Schedule =
3614 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3615 return Schedule == OMP_sch_static;
3616}
3617
3618bool CGOpenMPRuntime::isStaticNonchunked(
3619 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3620 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3621 return Schedule == OMP_dist_sch_static;
3622}
3623
3624bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3625 bool Chunked) const {
3626 OpenMPSchedType Schedule =
3627 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3628 return Schedule == OMP_sch_static_chunked;
3629}
3630
3631bool CGOpenMPRuntime::isStaticChunked(
3632 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3633 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3634 return Schedule == OMP_dist_sch_static_chunked;
3635}
3636
3637bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3638 OpenMPSchedType Schedule =
3639 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3640 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3640, __PRETTY_FUNCTION__))
;
3641 return Schedule != OMP_sch_static;
3642}
3643
3644static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3645 OpenMPScheduleClauseModifier M1,
3646 OpenMPScheduleClauseModifier M2) {
3647 int Modifier = 0;
3648 switch (M1) {
3649 case OMPC_SCHEDULE_MODIFIER_monotonic:
3650 Modifier = OMP_sch_modifier_monotonic;
3651 break;
3652 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3653 Modifier = OMP_sch_modifier_nonmonotonic;
3654 break;
3655 case OMPC_SCHEDULE_MODIFIER_simd:
3656 if (Schedule == OMP_sch_static_chunked)
3657 Schedule = OMP_sch_static_balanced_chunked;
3658 break;
3659 case OMPC_SCHEDULE_MODIFIER_last:
3660 case OMPC_SCHEDULE_MODIFIER_unknown:
3661 break;
3662 }
3663 switch (M2) {
3664 case OMPC_SCHEDULE_MODIFIER_monotonic:
3665 Modifier = OMP_sch_modifier_monotonic;
3666 break;
3667 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3668 Modifier = OMP_sch_modifier_nonmonotonic;
3669 break;
3670 case OMPC_SCHEDULE_MODIFIER_simd:
3671 if (Schedule == OMP_sch_static_chunked)
3672 Schedule = OMP_sch_static_balanced_chunked;
3673 break;
3674 case OMPC_SCHEDULE_MODIFIER_last:
3675 case OMPC_SCHEDULE_MODIFIER_unknown:
3676 break;
3677 }
3678 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3679 // If the static schedule kind is specified or if the ordered clause is
3680 // specified, and if the nonmonotonic modifier is not specified, the effect is
3681 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3682 // modifier is specified, the effect is as if the nonmonotonic modifier is
3683 // specified.
3684 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3685 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3686 Schedule == OMP_sch_static_balanced_chunked ||
3687 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
3688 Schedule == OMP_dist_sch_static_chunked ||
3689 Schedule == OMP_dist_sch_static))
3690 Modifier = OMP_sch_modifier_nonmonotonic;
3691 }
3692 return Schedule | Modifier;
3693}
3694
3695void CGOpenMPRuntime::emitForDispatchInit(
3696 CodeGenFunction &CGF, SourceLocation Loc,
3697 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3698 bool Ordered, const DispatchRTInput &DispatchValues) {
3699 if (!CGF.HaveInsertPoint())
3700 return;
3701 OpenMPSchedType Schedule = getRuntimeSchedule(
3702 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3703 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3704 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3705 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
3706 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 // Call __kmpc_dispatch_init(
3708 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3709 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3710 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3711
3712 // If the Chunk was not specified in the clause - use default value 1.
3713 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3714 : CGF.Builder.getIntN(IVSize, 1);
3715 llvm::Value *Args[] = {
3716 emitUpdateLocation(CGF, Loc),
3717 getThreadID(CGF, Loc),
3718 CGF.Builder.getInt32(addMonoNonMonoModifier(
3719 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3720 DispatchValues.LB, // Lower
3721 DispatchValues.UB, // Upper
3722 CGF.Builder.getIntN(IVSize, 1), // Stride
3723 Chunk // Chunk
3724 };
3725 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3726}
3727
3728static void emitForStaticInitCall(
3729 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3730 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3731 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3732 const CGOpenMPRuntime::StaticRTInput &Values) {
3733 if (!CGF.HaveInsertPoint())
3734 return;
3735
3736 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3736, __PRETTY_FUNCTION__))
;
3737 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3741, __PRETTY_FUNCTION__))
3738 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3741, __PRETTY_FUNCTION__))
3739 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3741, __PRETTY_FUNCTION__))
3740 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3741, __PRETTY_FUNCTION__))
3741 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3741, __PRETTY_FUNCTION__))
;
3742
3743 // Call __kmpc_for_static_init(
3744 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3745 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3746 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3747 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3748 llvm::Value *Chunk = Values.Chunk;
3749 if (Chunk == nullptr) {
3750 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3752, __PRETTY_FUNCTION__))
3751 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3752, __PRETTY_FUNCTION__))
3752 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3752, __PRETTY_FUNCTION__))
;
3753 // If the Chunk was not specified in the clause - use default value 1.
3754 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3755 } else {
3756 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3760, __PRETTY_FUNCTION__))
3757 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3760, __PRETTY_FUNCTION__))
3758 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3760, __PRETTY_FUNCTION__))
3759 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3760, __PRETTY_FUNCTION__))
3760 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3760, __PRETTY_FUNCTION__))
;
3761 }
3762 llvm::Value *Args[] = {
3763 UpdateLocation,
3764 ThreadId,
3765 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3766 M2)), // Schedule type
3767 Values.IL.getPointer(), // &isLastIter
3768 Values.LB.getPointer(), // &LB
3769 Values.UB.getPointer(), // &UB
3770 Values.ST.getPointer(), // &Stride
3771 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3772 Chunk // Chunk
3773 };
3774 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3775}
3776
3777void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3778 SourceLocation Loc,
3779 OpenMPDirectiveKind DKind,
3780 const OpenMPScheduleTy &ScheduleKind,
3781 const StaticRTInput &Values) {
3782 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3783 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3784 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3785, __PRETTY_FUNCTION__))
3785 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3785, __PRETTY_FUNCTION__))
;
3786 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3787 isOpenMPLoopDirective(DKind)
3788 ? OMP_IDENT_WORK_LOOP
3789 : OMP_IDENT_WORK_SECTIONS);
3790 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3791 llvm::FunctionCallee StaticInitFunction =
3792 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3793 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3794 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3795 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3796}
3797
3798void CGOpenMPRuntime::emitDistributeStaticInit(
3799 CodeGenFunction &CGF, SourceLocation Loc,
3800 OpenMPDistScheduleClauseKind SchedKind,
3801 const CGOpenMPRuntime::StaticRTInput &Values) {
3802 OpenMPSchedType ScheduleNum =
3803 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3804 llvm::Value *UpdatedLocation =
3805 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3806 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3807 llvm::FunctionCallee StaticInitFunction =
3808 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3809 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3810 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3811 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3812}
3813
3814void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3815 SourceLocation Loc,
3816 OpenMPDirectiveKind DKind) {
3817 if (!CGF.HaveInsertPoint())
3818 return;
3819 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3820 llvm::Value *Args[] = {
3821 emitUpdateLocation(CGF, Loc,
3822 isOpenMPDistributeDirective(DKind)
3823 ? OMP_IDENT_WORK_DISTRIBUTE
3824 : isOpenMPLoopDirective(DKind)
3825 ? OMP_IDENT_WORK_LOOP
3826 : OMP_IDENT_WORK_SECTIONS),
3827 getThreadID(CGF, Loc)};
3828 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3829 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3830 Args);
3831}
3832
3833void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3834 SourceLocation Loc,
3835 unsigned IVSize,
3836 bool IVSigned) {
3837 if (!CGF.HaveInsertPoint())
3838 return;
3839 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3840 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3841 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3842}
3843
3844llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3845 SourceLocation Loc, unsigned IVSize,
3846 bool IVSigned, Address IL,
3847 Address LB, Address UB,
3848 Address ST) {
3849 // Call __kmpc_dispatch_next(
3850 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3851 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3852 // kmp_int[32|64] *p_stride);
3853 llvm::Value *Args[] = {
3854 emitUpdateLocation(CGF, Loc),
3855 getThreadID(CGF, Loc),
3856 IL.getPointer(), // &isLastIter
3857 LB.getPointer(), // &Lower
3858 UB.getPointer(), // &Upper
3859 ST.getPointer() // &Stride
3860 };
3861 llvm::Value *Call =
3862 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3863 return CGF.EmitScalarConversion(
3864 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3865 CGF.getContext().BoolTy, Loc);
3866}
3867
3868void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3869 llvm::Value *NumThreads,
3870 SourceLocation Loc) {
3871 if (!CGF.HaveInsertPoint())
3872 return;
3873 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3874 llvm::Value *Args[] = {
3875 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3876 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3877 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3878 Args);
3879}
3880
3881void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3882 ProcBindKind ProcBind,
3883 SourceLocation Loc) {
3884 if (!CGF.HaveInsertPoint())
3885 return;
3886 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3886, __PRETTY_FUNCTION__))
;
3887 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3888 llvm::Value *Args[] = {
3889 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3890 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
3891 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3892}
3893
3894void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3895 SourceLocation Loc, llvm::AtomicOrdering AO) {
3896 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3897 if (OMPBuilder) {
3898 OMPBuilder->CreateFlush(CGF.Builder);
3899 } else {
3900 if (!CGF.HaveInsertPoint())
3901 return;
3902 // Build call void __kmpc_flush(ident_t *loc)
3903 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3904 emitUpdateLocation(CGF, Loc));
3905 }
3906}
3907
3908namespace {
3909/// Indexes of fields for type kmp_task_t.
3910enum KmpTaskTFields {
3911 /// List of shared variables.
3912 KmpTaskTShareds,
3913 /// Task routine.
3914 KmpTaskTRoutine,
3915 /// Partition id for the untied tasks.
3916 KmpTaskTPartId,
3917 /// Function with call of destructors for private variables.
3918 Data1,
3919 /// Task priority.
3920 Data2,
3921 /// (Taskloops only) Lower bound.
3922 KmpTaskTLowerBound,
3923 /// (Taskloops only) Upper bound.
3924 KmpTaskTUpperBound,
3925 /// (Taskloops only) Stride.
3926 KmpTaskTStride,
3927 /// (Taskloops only) Is last iteration flag.
3928 KmpTaskTLastIter,
3929 /// (Taskloops only) Reduction data.
3930 KmpTaskTReductions,
3931};
3932} // anonymous namespace
3933
3934bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3935 return OffloadEntriesTargetRegion.empty() &&
3936 OffloadEntriesDeviceGlobalVar.empty();
3937}
3938
3939/// Initialize target region entry.
3940void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3941 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3942 StringRef ParentName, unsigned LineNum,
3943 unsigned Order) {
3944 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3946, __PRETTY_FUNCTION__))
3945 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3946, __PRETTY_FUNCTION__))
3946 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3946, __PRETTY_FUNCTION__))
;
3947 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3948 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3949 OMPTargetRegionEntryTargetRegion);
3950 ++OffloadingEntriesNum;
3951}
3952
3953void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3954 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3955 StringRef ParentName, unsigned LineNum,
3956 llvm::Constant *Addr, llvm::Constant *ID,
3957 OMPTargetRegionEntryKind Flags) {
3958 // If we are emitting code for a target, the entry is already initialized,
3959 // only has to be registered.
3960 if (CGM.getLangOpts().OpenMPIsDevice) {
3961 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3962 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3963 DiagnosticsEngine::Error,
3964 "Unable to find target region on line '%0' in the device code.");
3965 CGM.getDiags().Report(DiagID) << LineNum;
3966 return;
3967 }
3968 auto &Entry =
3969 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3970 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3970, __PRETTY_FUNCTION__))
;
3971 Entry.setAddress(Addr);
3972 Entry.setID(ID);
3973 Entry.setFlags(Flags);
3974 } else {
3975 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3976 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3977 ++OffloadingEntriesNum;
3978 }
3979}
3980
3981bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3982 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3983 unsigned LineNum) const {
3984 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3985 if (PerDevice == OffloadEntriesTargetRegion.end())
3986 return false;
3987 auto PerFile = PerDevice->second.find(FileID);
3988 if (PerFile == PerDevice->second.end())
3989 return false;
3990 auto PerParentName = PerFile->second.find(ParentName);
3991 if (PerParentName == PerFile->second.end())
3992 return false;
3993 auto PerLine = PerParentName->second.find(LineNum);
3994 if (PerLine == PerParentName->second.end())
3995 return false;
3996 // Fail if this entry is already registered.
3997 if (PerLine->second.getAddress() || PerLine->second.getID())
3998 return false;
3999 return true;
4000}
4001
4002void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
4003 const OffloadTargetRegionEntryInfoActTy &Action) {
4004 // Scan all target region entries and perform the provided action.
4005 for (const auto &D : OffloadEntriesTargetRegion)
4006 for (const auto &F : D.second)
4007 for (const auto &P : F.second)
4008 for (const auto &L : P.second)
4009 Action(D.first, F.first, P.first(), L.first, L.second);
4010}
4011
4012void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4013 initializeDeviceGlobalVarEntryInfo(StringRef Name,
4014 OMPTargetGlobalVarEntryKind Flags,
4015 unsigned Order) {
4016 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4018, __PRETTY_FUNCTION__))
4017 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4018, __PRETTY_FUNCTION__))
4018 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4018, __PRETTY_FUNCTION__))
;
4019 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
4020 ++OffloadingEntriesNum;
4021}
4022
4023void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4024 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
4025 CharUnits VarSize,
4026 OMPTargetGlobalVarEntryKind Flags,
4027 llvm::GlobalValue::LinkageTypes Linkage) {
4028 if (CGM.getLangOpts().OpenMPIsDevice) {
4029 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4030 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4031, __PRETTY_FUNCTION__))
4031 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4031, __PRETTY_FUNCTION__))
;
4032 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4033, __PRETTY_FUNCTION__))
4033 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4033, __PRETTY_FUNCTION__))
;
4034 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
4035 if (Entry.getVarSize().isZero()) {
4036 Entry.setVarSize(VarSize);
4037 Entry.setLinkage(Linkage);
4038 }
4039 return;
4040 }
4041 Entry.setVarSize(VarSize);
4042 Entry.setLinkage(Linkage);
4043 Entry.setAddress(Addr);
4044 } else {
4045 if (hasDeviceGlobalVarEntryInfo(VarName)) {
4046 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4047 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4048, __PRETTY_FUNCTION__))
4048 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4048, __PRETTY_FUNCTION__))
;
4049 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4050, __PRETTY_FUNCTION__))
4050 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4050, __PRETTY_FUNCTION__))
;
4051 if (Entry.getVarSize().isZero()) {
4052 Entry.setVarSize(VarSize);
4053 Entry.setLinkage(Linkage);
4054 }
4055 return;
4056 }
4057 OffloadEntriesDeviceGlobalVar.try_emplace(
4058 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4059 ++OffloadingEntriesNum;
4060 }
4061}
4062
4063void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4064 actOnDeviceGlobalVarEntriesInfo(
4065 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4066 // Scan all target region entries and perform the provided action.
4067 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4068 Action(E.getKey(), E.getValue());
4069}
4070
4071void CGOpenMPRuntime::createOffloadEntry(
4072 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4073 llvm::GlobalValue::LinkageTypes Linkage) {
4074 StringRef Name = Addr->getName();
4075 llvm::Module &M = CGM.getModule();
4076 llvm::LLVMContext &C = M.getContext();
4077
4078 // Create constant string with the name.
4079 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4080
4081 std::string StringName = getName({"omp_offloading", "entry_name"});
4082 auto *Str = new llvm::GlobalVariable(
4083 M, StrPtrInit->getType(), /*isConstant=*/true,
4084 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4085 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4086
4087 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4088 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4089 llvm::ConstantInt::get(CGM.SizeTy, Size),
4090 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4091 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4092 std::string EntryName = getName({"omp_offloading", "entry", ""});
4093 llvm::GlobalVariable *Entry = createGlobalStruct(
4094 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4095 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4096
4097 // The entry has to be created in the section the linker expects it to be.
4098 Entry->setSection("omp_offloading_entries");
4099}
4100
4101void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4102 // Emit the offloading entries and metadata so that the device codegen side
4103 // can easily figure out what to emit. The produced metadata looks like
4104 // this:
4105 //
4106 // !omp_offload.info = !{!1, ...}
4107 //
4108 // Right now we only generate metadata for function that contain target
4109 // regions.
4110
4111 // If we are in simd mode or there are no entries, we don't need to do
4112 // anything.
4113 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
4114 return;
4115
4116 llvm::Module &M = CGM.getModule();
4117 llvm::LLVMContext &C = M.getContext();
4118 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4119 SourceLocation, StringRef>,
4120 16>
4121 OrderedEntries(OffloadEntriesInfoManager.size());
4122 llvm::SmallVector<StringRef, 16> ParentFunctions(
4123 OffloadEntriesInfoManager.size());
4124
4125 // Auxiliary methods to create metadata values and strings.
4126 auto &&GetMDInt = [this](unsigned V) {
4127 return llvm::ConstantAsMetadata::get(
4128 llvm::ConstantInt::get(CGM.Int32Ty, V));
4129 };
4130
4131 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4132
4133 // Create the offloading info metadata node.
4134 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4135
4136 // Create function that emits metadata for each target region entry;
4137 auto &&TargetRegionMetadataEmitter =
4138 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4139 &GetMDString](
4140 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4141 unsigned Line,
4142 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4143 // Generate metadata for target regions. Each entry of this metadata
4144 // contains:
4145 // - Entry 0 -> Kind of this type of metadata (0).
4146 // - Entry 1 -> Device ID of the file where the entry was identified.
4147 // - Entry 2 -> File ID of the file where the entry was identified.
4148 // - Entry 3 -> Mangled name of the function where the entry was
4149 // identified.
4150 // - Entry 4 -> Line in the file where the entry was identified.
4151 // - Entry 5 -> Order the entry was created.
4152 // The first element of the metadata node is the kind.
4153 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4154 GetMDInt(FileID), GetMDString(ParentName),
4155 GetMDInt(Line), GetMDInt(E.getOrder())};
4156
4157 SourceLocation Loc;
4158 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4159 E = CGM.getContext().getSourceManager().fileinfo_end();
4160 I != E; ++I) {
4161 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4162 I->getFirst()->getUniqueID().getFile() == FileID) {
4163 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4164 I->getFirst(), Line, 1);
4165 break;
4166 }
4167 }
4168 // Save this entry in the right position of the ordered entries array.
4169 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4170 ParentFunctions[E.getOrder()] = ParentName;
4171
4172 // Add metadata to the named metadata node.
4173 MD->addOperand(llvm::MDNode::get(C, Ops));
4174 };
4175
4176 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4177 TargetRegionMetadataEmitter);
4178
4179 // Create function that emits metadata for each device global variable entry;
4180 auto &&DeviceGlobalVarMetadataEmitter =
4181 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4182 MD](StringRef MangledName,
4183 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4184 &E) {
4185 // Generate metadata for global variables. Each entry of this metadata
4186 // contains:
4187 // - Entry 0 -> Kind of this type of metadata (1).
4188 // - Entry 1 -> Mangled name of the variable.
4189 // - Entry 2 -> Declare target kind.
4190 // - Entry 3 -> Order the entry was created.
4191 // The first element of the metadata node is the kind.
4192 llvm::Metadata *Ops[] = {
4193 GetMDInt(E.getKind()), GetMDString(MangledName),
4194 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4195
4196 // Save this entry in the right position of the ordered entries array.
4197 OrderedEntries[E.getOrder()] =
4198 std::make_tuple(&E, SourceLocation(), MangledName);
4199
4200 // Add metadata to the named metadata node.
4201 MD->addOperand(llvm::MDNode::get(C, Ops));
4202 };
4203
4204 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4205 DeviceGlobalVarMetadataEmitter);
4206
4207 for (const auto &E : OrderedEntries) {
4208 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4208, __PRETTY_FUNCTION__))
;
4209 if (const auto *CE =
4210 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4211 std::get<0>(E))) {
4212 if (!CE->getID() || !CE->getAddress()) {
4213 // Do not blame the entry if the parent funtion is not emitted.
4214 StringRef FnName = ParentFunctions[CE->getOrder()];
4215 if (!CGM.GetGlobalValue(FnName))
4216 continue;
4217 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4218 DiagnosticsEngine::Error,
4219 "Offloading entry for target region in %0 is incorrect: either the "
4220 "address or the ID is invalid.");
4221 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4222 continue;
4223 }
4224 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4225 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4226 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
4227 OffloadEntryInfoDeviceGlobalVar>(
4228 std::get<0>(E))) {
4229 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4230 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4231 CE->getFlags());
4232 switch (Flags) {
4233 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4234 if (CGM.getLangOpts().OpenMPIsDevice &&
4235 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4236 continue;
4237 if (!CE->getAddress()) {
4238 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4239 DiagnosticsEngine::Error, "Offloading entry for declare target "
4240 "variable %0 is incorrect: the "
4241 "address is invalid.");
4242 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4243 continue;
4244 }
4245 // The vaiable has no definition - no need to add the entry.
4246 if (CE->getVarSize().isZero())
4247 continue;
4248 break;
4249 }
4250 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4251 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4253, __PRETTY_FUNCTION__))
4252 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4253, __PRETTY_FUNCTION__))
4253 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4253, __PRETTY_FUNCTION__))
;
4254 if (CGM.getLangOpts().OpenMPIsDevice)
4255 continue;
4256 if (!CE->getAddress()) {
4257 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4258 DiagnosticsEngine::Error,
4259 "Offloading entry for declare target variable is incorrect: the "
4260 "address is invalid.");
4261 CGM.getDiags().Report(DiagID);
4262 continue;
4263 }
4264 break;
4265 }
4266 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4267 CE->getVarSize().getQuantity(), Flags,
4268 CE->getLinkage());
4269 } else {
4270 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4270)
;
4271 }
4272 }
4273}
4274
4275/// Loads all the offload entries information from the host IR
4276/// metadata.
4277void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4278 // If we are in target mode, load the metadata from the host IR. This code has
4279 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4280
4281 if (!CGM.getLangOpts().OpenMPIsDevice)
4282 return;
4283
4284 if (CGM.getLangOpts().OMPHostIRFile.empty())
4285 return;
4286
4287 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4288 if (auto EC = Buf.getError()) {
4289 CGM.getDiags().Report(diag::err_cannot_open_file)
4290 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4291 return;
4292 }
4293
4294 llvm::LLVMContext C;
4295 auto ME = expectedToErrorOrAndEmitErrors(
4296 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4297
4298 if (auto EC = ME.getError()) {
4299 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4300 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4301 CGM.getDiags().Report(DiagID)
4302 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4303 return;
4304 }
4305
4306 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4307 if (!MD)
4308 return;
4309
4310 for (llvm::MDNode *MN : MD->operands()) {
4311 auto &&GetMDInt = [MN](unsigned Idx) {
4312 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4313 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4314 };
4315
4316 auto &&GetMDString = [MN](unsigned Idx) {
4317 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4318 return V->getString();
4319 };
4320
4321 switch (GetMDInt(0)) {
4322 default:
4323 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4323)
;
4324 break;
4325 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4326 OffloadingEntryInfoTargetRegion:
4327 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4328 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4329 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4330 /*Order=*/GetMDInt(5));
4331 break;
4332 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4333 OffloadingEntryInfoDeviceGlobalVar:
4334 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4335 /*MangledName=*/GetMDString(1),
4336 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4337 /*Flags=*/GetMDInt(2)),
4338 /*Order=*/GetMDInt(3));
4339 break;
4340 }
4341 }
4342}
4343
4344void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4345 if (!KmpRoutineEntryPtrTy) {
4346 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4347 ASTContext &C = CGM.getContext();
4348 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4349 FunctionProtoType::ExtProtoInfo EPI;
4350 KmpRoutineEntryPtrQTy = C.getPointerType(
4351 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4352 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4353 }
4354}
4355
4356QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4357 // Make sure the type of the entry is already created. This is the type we
4358 // have to create:
4359 // struct __tgt_offload_entry{
4360 // void *addr; // Pointer to the offload entry info.
4361 // // (function or global)
4362 // char *name; // Name of the function or global.
4363 // size_t size; // Size of the entry info (0 if it a function).
4364 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4365 // int32_t reserved; // Reserved, to use by the runtime library.
4366 // };
4367 if (TgtOffloadEntryQTy.isNull()) {
4368 ASTContext &C = CGM.getContext();
4369 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4370 RD->startDefinition();
4371 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4372 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4373 addFieldToRecordDecl(C, RD, C.getSizeType());
4374 addFieldToRecordDecl(
4375 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4376 addFieldToRecordDecl(
4377 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4378 RD->completeDefinition();
4379 RD->addAttr(PackedAttr::CreateImplicit(C));
4380 TgtOffloadEntryQTy = C.getRecordType(RD);
4381 }
4382 return TgtOffloadEntryQTy;
4383}
4384
4385namespace {
4386struct PrivateHelpersTy {
4387 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4388 const VarDecl *PrivateElemInit)
4389 : Original(Original), PrivateCopy(PrivateCopy),
4390 PrivateElemInit(PrivateElemInit) {}
4391 const VarDecl *Original;
4392 const VarDecl *PrivateCopy;
4393 const VarDecl *PrivateElemInit;
4394};
4395typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4396} // anonymous namespace
4397
4398static RecordDecl *
4399createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4400 if (!Privates.empty()) {
4401 ASTContext &C = CGM.getContext();
4402 // Build struct .kmp_privates_t. {
4403 // /* private vars */
4404 // };
4405 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4406 RD->startDefinition();
4407 for (const auto &Pair : Privates) {
4408 const VarDecl *VD = Pair.second.Original;
4409 QualType Type = VD->getType().getNonReferenceType();
4410 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4411 if (VD->hasAttrs()) {
4412 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4413 E(VD->getAttrs().end());
4414 I != E; ++I)
4415 FD->addAttr(*I);
4416 }
4417 }
4418 RD->completeDefinition();
4419 return RD;
4420 }
4421 return nullptr;
4422}
4423
4424static RecordDecl *
4425createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4426 QualType KmpInt32Ty,
4427 QualType KmpRoutineEntryPointerQTy) {
4428 ASTContext &C = CGM.getContext();
4429 // Build struct kmp_task_t {
4430 // void * shareds;
4431 // kmp_routine_entry_t routine;
4432 // kmp_int32 part_id;
4433 // kmp_cmplrdata_t data1;
4434 // kmp_cmplrdata_t data2;
4435 // For taskloops additional fields:
4436 // kmp_uint64 lb;
4437 // kmp_uint64 ub;
4438 // kmp_int64 st;
4439 // kmp_int32 liter;
4440 // void * reductions;
4441 // };
4442 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4443 UD->startDefinition();
4444 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4445 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4446 UD->completeDefinition();
4447 QualType KmpCmplrdataTy = C.getRecordType(UD);
4448 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4449 RD->startDefinition();
4450 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4451 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4452 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4453 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4454 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4455 if (isOpenMPTaskLoopDirective(Kind)) {
4456 QualType KmpUInt64Ty =
4457 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4458 QualType KmpInt64Ty =
4459 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4460 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4461 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4462 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4463 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4464 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4465 }
4466 RD->completeDefinition();
4467 return RD;
4468}
4469
4470static RecordDecl *
4471createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4472 ArrayRef<PrivateDataTy> Privates) {
4473 ASTContext &C = CGM.getContext();
4474 // Build struct kmp_task_t_with_privates {
4475 // kmp_task_t task_data;
4476 // .kmp_privates_t. privates;
4477 // };
4478 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4479 RD->startDefinition();
4480 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4481 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4482 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4483 RD->completeDefinition();
4484 return RD;
4485}
4486
4487/// Emit a proxy function which accepts kmp_task_t as the second
4488/// argument.
4489/// \code
4490/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4491/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4492/// For taskloops:
4493/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4494/// tt->reductions, tt->shareds);
4495/// return 0;
4496/// }
4497/// \endcode
4498static llvm::Function *
4499emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4500 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4501 QualType KmpTaskTWithPrivatesPtrQTy,
4502 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4503 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4504 llvm::Value *TaskPrivatesMap) {
4505 ASTContext &C = CGM.getContext();
4506 FunctionArgList Args;
4507 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4508 ImplicitParamDecl::Other);
4509 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4510 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4511 ImplicitParamDecl::Other);
4512 Args.push_back(&GtidArg);
4513 Args.push_back(&TaskTypeArg);
4514 const auto &TaskEntryFnInfo =
4515 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4516 llvm::FunctionType *TaskEntryTy =
4517 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4518 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4519 auto *TaskEntry = llvm::Function::Create(
4520 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4521 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4522 TaskEntry->setDoesNotRecurse();
4523 CodeGenFunction CGF(CGM);
4524 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4525 Loc, Loc);
4526
4527 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4528 // tt,
4529 // For taskloops:
4530 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4531 // tt->task_data.shareds);
4532 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4533 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4534 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4535 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4536 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4537 const auto *KmpTaskTWithPrivatesQTyRD =
4538 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4539 LValue Base =
4540 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4541 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4542 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4543 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4544 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4545
4546 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4547 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4548 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4549 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4550 CGF.ConvertTypeForMem(SharedsPtrTy));
4551
4552 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4553 llvm::Value *PrivatesParam;
4554 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4555 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4556 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4557 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
4558 } else {
4559 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4560 }
4561
4562 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4563 TaskPrivatesMap,
4564 CGF.Builder
4565 .CreatePointerBitCastOrAddrSpaceCast(
4566 TDBase.getAddress(CGF), CGF.VoidPtrTy)
4567 .getPointer()};
4568 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4569 std::end(CommonArgs));
4570 if (isOpenMPTaskLoopDirective(Kind)) {
4571 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4572 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4573 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4574 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4575 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4576 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4577 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4578 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4579 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4580 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4581 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4582 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4583 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4584 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4585 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4586 CallArgs.push_back(LBParam);
4587 CallArgs.push_back(UBParam);
4588 CallArgs.push_back(StParam);
4589 CallArgs.push_back(LIParam);
4590 CallArgs.push_back(RParam);
4591 }
4592 CallArgs.push_back(SharedsParam);
4593
4594 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4595 CallArgs);
4596 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4597 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4598 CGF.FinishFunction();
4599 return TaskEntry;
4600}
4601
4602static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4603 SourceLocation Loc,
4604 QualType KmpInt32Ty,
4605 QualType KmpTaskTWithPrivatesPtrQTy,
4606 QualType KmpTaskTWithPrivatesQTy) {
4607 ASTContext &C = CGM.getContext();
4608 FunctionArgList Args;
4609 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4610 ImplicitParamDecl::Other);
4611 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4612 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4613 ImplicitParamDecl::Other);
4614 Args.push_back(&GtidArg);
4615 Args.push_back(&TaskTypeArg);
4616 const auto &DestructorFnInfo =
4617 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4618 llvm::FunctionType *DestructorFnTy =
4619 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4620 std::string Name =
4621 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4622 auto *DestructorFn =
4623 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4624 Name, &CGM.getModule());
4625 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4626 DestructorFnInfo);
4627 DestructorFn->setDoesNotRecurse();
4628 CodeGenFunction CGF(CGM);
4629 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4630 Args, Loc, Loc);
4631
4632 LValue Base = CGF.EmitLoadOfPointerLValue(
4633 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4634 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4635 const auto *KmpTaskTWithPrivatesQTyRD =
4636 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4637 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4638 Base = CGF.EmitLValueForField(Base, *FI);
4639 for (const auto *Field :
4640 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4641 if (QualType::DestructionKind DtorKind =
4642 Field->getType().isDestructedType()) {
4643 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4644 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
4645 }
4646 }
4647 CGF.FinishFunction();
4648 return DestructorFn;
4649}
4650
4651/// Emit a privates mapping function for correct handling of private and
4652/// firstprivate variables.
4653/// \code
4654/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4655/// **noalias priv1,..., <tyn> **noalias privn) {
4656/// *priv1 = &.privates.priv1;
4657/// ...;
4658/// *privn = &.privates.privn;
4659/// }
4660/// \endcode
4661static llvm::Value *
4662emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4663 ArrayRef<const Expr *> PrivateVars,
4664 ArrayRef<const Expr *> FirstprivateVars,
4665 ArrayRef<const Expr *> LastprivateVars,
4666 QualType PrivatesQTy,
4667 ArrayRef<PrivateDataTy> Privates) {
4668 ASTContext &C = CGM.getContext();
4669 FunctionArgList Args;
4670 ImplicitParamDecl TaskPrivatesArg(
4671 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4672 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4673 ImplicitParamDecl::Other);
4674 Args.push_back(&TaskPrivatesArg);
4675 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4676 unsigned Counter = 1;
4677 for (const Expr *E : PrivateVars) {
4678 Args.push_back(ImplicitParamDecl::Create(
4679 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4680 C.getPointerType(C.getPointerType(E->getType()))
4681 .withConst()
4682 .withRestrict(),
4683 ImplicitParamDecl::Other));
4684 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4685 PrivateVarsPos[VD] = Counter;
4686 ++Counter;
4687 }
4688 for (const Expr *E : FirstprivateVars) {
4689 Args.push_back(ImplicitParamDecl::Create(
4690 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4691 C.getPointerType(C.getPointerType(E->getType()))
4692 .withConst()
4693 .withRestrict(),
4694 ImplicitParamDecl::Other));
4695 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4696 PrivateVarsPos[VD] = Counter;
4697 ++Counter;
4698 }
4699 for (const Expr *E : LastprivateVars) {
4700 Args.push_back(ImplicitParamDecl::Create(
4701 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4702 C.getPointerType(C.getPointerType(E->getType()))
4703 .withConst()
4704 .withRestrict(),
4705 ImplicitParamDecl::Other));
4706 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4707 PrivateVarsPos[VD] = Counter;
4708 ++Counter;
4709 }
4710 const auto &TaskPrivatesMapFnInfo =
4711 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4712 llvm::FunctionType *TaskPrivatesMapTy =
4713 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4714 std::string Name =
4715 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4716 auto *TaskPrivatesMap = llvm::Function::Create(
4717 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4718 &CGM.getModule());
4719 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4720 TaskPrivatesMapFnInfo);
4721 if (CGM.getLangOpts().Optimize) {
4722 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4723 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4724 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4725 }
4726 CodeGenFunction CGF(CGM);
4727 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4728 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4729
4730 // *privi = &.privates.privi;
4731 LValue Base = CGF.EmitLoadOfPointerLValue(
4732 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4733 TaskPrivatesArg.getType()->castAs<PointerType>());
4734 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4735 Counter = 0;
4736 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4737 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4738 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4739 LValue RefLVal =
4740 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4741 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4742 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
4743 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
4744 ++Counter;
4745 }
4746 CGF.FinishFunction();
4747 return TaskPrivatesMap;
4748}
4749
4750/// Emit initialization for private variables in task-based directives.
4751static void emitPrivatesInit(CodeGenFunction &CGF,
4752 const OMPExecutableDirective &D,
4753 Address KmpTaskSharedsPtr, LValue TDBase,
4754 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4755 QualType SharedsTy, QualType SharedsPtrTy,
4756 const OMPTaskDataTy &Data,
4757 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4758 ASTContext &C = CGF.getContext();
4759 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4760 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4761 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4762 ? OMPD_taskloop
4763 : OMPD_task;
4764 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4765 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4766 LValue SrcBase;
4767 bool IsTargetTask =
4768 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4769 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4770 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4771 // PointersArray and SizesArray. The original variables for these arrays are
4772 // not captured and we get their addresses explicitly.
4773 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4774 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4775 SrcBase = CGF.MakeAddrLValue(
4776 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4777 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4778 SharedsTy);
4779 }
4780 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4781 for (const PrivateDataTy &Pair : Privates) {
4782 const VarDecl *VD = Pair.second.PrivateCopy;
4783 const Expr *Init = VD->getAnyInitializer();
4784 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4785 !CGF.isTrivialInitializer(Init)))) {
4786 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4787 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4788 const VarDecl *OriginalVD = Pair.second.Original;
4789 // Check if the variable is the target-based BasePointersArray,
4790 // PointersArray or SizesArray.
4791 LValue SharedRefLValue;
4792 QualType Type = PrivateLValue.getType();
4793 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4794 if (IsTargetTask && !SharedField) {
4795 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4796 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4797 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4798 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4799 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4800 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4801 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
4802 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4802, __PRETTY_FUNCTION__))
;
4803 SharedRefLValue =
4804 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4805 } else {
4806 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4807 SharedRefLValue = CGF.MakeAddrLValue(
4808 Address(SharedRefLValue.getPointer(CGF),
4809 C.getDeclAlign(OriginalVD)),
4810 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4811 SharedRefLValue.getTBAAInfo());
4812 }
4813 if (Type->isArrayType()) {
4814 // Initialize firstprivate array.
4815 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4816 // Perform simple memcpy.
4817 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4818 } else {
4819 // Initialize firstprivate array using element-by-element
4820 // initialization.
4821 CGF.EmitOMPAggregateAssign(
4822 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
4823 Type,
4824 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4825 Address SrcElement) {
4826 // Clean up any temporaries needed by the initialization.
4827 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4828 InitScope.addPrivate(
4829 Elem, [SrcElement]() -> Address { return SrcElement; });
4830 (void)InitScope.Privatize();
4831 // Emit initialization for single element.
4832 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4833 CGF, &CapturesInfo);
4834 CGF.EmitAnyExprToMem(Init, DestElement,
4835 Init->getType().getQualifiers(),
4836 /*IsInitializer=*/false);
4837 });
4838 }
4839 } else {
4840 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4841 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
4842 return SharedRefLValue.getAddress(CGF);
4843 });
4844 (void)InitScope.Privatize();
4845 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4846 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4847 /*capturedByInit=*/false);
4848 }
4849 } else {
4850 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4851 }
4852 }
4853 ++FI;
4854 }
4855}
4856
4857/// Check if duplication function is required for taskloops.
4858static bool checkInitIsRequired(CodeGenFunction &CGF,
4859 ArrayRef<PrivateDataTy> Privates) {
4860 bool InitRequired = false;
4861 for (const PrivateDataTy &Pair : Privates) {
4862 const VarDecl *VD = Pair.second.PrivateCopy;
4863 const Expr *Init = VD->getAnyInitializer();
4864 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4865 !CGF.isTrivialInitializer(Init));
4866 if (InitRequired)
4867 break;
4868 }
4869 return InitRequired;
4870}
4871
4872
4873/// Emit task_dup function (for initialization of
4874/// private/firstprivate/lastprivate vars and last_iter flag)
4875/// \code
4876/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4877/// lastpriv) {
4878/// // setup lastprivate flag
4879/// task_dst->last = lastpriv;
4880/// // could be constructor calls here...
4881/// }
4882/// \endcode
4883static llvm::Value *
4884emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4885 const OMPExecutableDirective &D,
4886 QualType KmpTaskTWithPrivatesPtrQTy,
4887 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4888 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4889 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4890 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4891 ASTContext &C = CGM.getContext();
4892 FunctionArgList Args;
4893 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4894 KmpTaskTWithPrivatesPtrQTy,
4895 ImplicitParamDecl::Other);
4896 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4897 KmpTaskTWithPrivatesPtrQTy,
4898 ImplicitParamDecl::Other);
4899 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4900 ImplicitParamDecl::Other);
4901 Args.push_back(&DstArg);
4902 Args.push_back(&SrcArg);
4903 Args.push_back(&LastprivArg);
4904 const auto &TaskDupFnInfo =
4905 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4906 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4907 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4908 auto *TaskDup = llvm::Function::Create(
4909 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4910 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4911 TaskDup->setDoesNotRecurse();
4912 CodeGenFunction CGF(CGM);
4913 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4914 Loc);
4915
4916 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4917 CGF.GetAddrOfLocalVar(&DstArg),
4918 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4919 // task_dst->liter = lastpriv;
4920 if (WithLastIter) {
4921 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4922 LValue Base = CGF.EmitLValueForField(
4923 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4924 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4925 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4926 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4927 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4928 }
4929
4930 // Emit initial values for private copies (if any).
4931 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4931, __PRETTY_FUNCTION__))
;
4932 Address KmpTaskSharedsPtr = Address::invalid();
4933 if (!Data.FirstprivateVars.empty()) {
4934 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4935 CGF.GetAddrOfLocalVar(&SrcArg),
4936 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4937 LValue Base = CGF.EmitLValueForField(
4938 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4939 KmpTaskSharedsPtr = Address(
4940 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4941 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4942 KmpTaskTShareds)),
4943 Loc),
4944 CGF.getNaturalTypeAlignment(SharedsTy));
4945 }
4946 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4947 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4948 CGF.FinishFunction();
4949 return TaskDup;
4950}
4951
4952/// Checks if destructor function is required to be generated.
4953/// \return true if cleanups are required, false otherwise.
4954static bool
4955checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4956 bool NeedsCleanup = false;
4957 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4958 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4959 for (const FieldDecl *FD : PrivateRD->fields()) {
4960 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4961 if (NeedsCleanup)
4962 break;
4963 }
4964 return NeedsCleanup;
4965}
4966
4967CGOpenMPRuntime::TaskResultTy
4968CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4969 const OMPExecutableDirective &D,
4970 llvm::Function *TaskFunction, QualType SharedsTy,
4971 Address Shareds, const OMPTaskDataTy &Data) {
4972 ASTContext &C = CGM.getContext();
4973 llvm::SmallVector<PrivateDataTy, 4> Privates;
4974 // Aggregate privates and sort them by the alignment.
4975 auto I = Data.PrivateCopies.begin();
4976 for (const Expr *E : Data.PrivateVars) {
4977 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4978 Privates.emplace_back(
4979 C.getDeclAlign(VD),
4980 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4981 /*PrivateElemInit=*/nullptr));
4982 ++I;
4983 }
4984 I = Data.FirstprivateCopies.begin();
4985 auto IElemInitRef = Data.FirstprivateInits.begin();
4986 for (const Expr *E : Data.FirstprivateVars) {
4987 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4988 Privates.emplace_back(
4989 C.getDeclAlign(VD),
4990 PrivateHelpersTy(
4991 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4992 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4993 ++I;
4994 ++IElemInitRef;
4995 }
4996 I = Data.LastprivateCopies.begin();
4997 for (const Expr *E : Data.LastprivateVars) {
4998 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4999 Privates.emplace_back(
5000 C.getDeclAlign(VD),
5001 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5002 /*PrivateElemInit=*/nullptr));
5003 ++I;
5004 }
5005 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5006 return L.first > R.first;
5007 });
5008 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5009 // Build type kmp_routine_entry_t (if not built yet).
5010 emitKmpRoutineEntryT(KmpInt32Ty);
5011 // Build type kmp_task_t (if not built yet).
5012 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5013 if (SavedKmpTaskloopTQTy.isNull()) {
5014 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5015 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5016 }
5017 KmpTaskTQTy = SavedKmpTaskloopTQTy;
5018 } else {
5019 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5022, __PRETTY_FUNCTION__))
5020 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5022, __PRETTY_FUNCTION__))
5021 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5022, __PRETTY_FUNCTION__))
5022 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5022, __PRETTY_FUNCTION__))
;
5023 if (SavedKmpTaskTQTy.isNull()) {
5024 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5025 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5026 }
5027 KmpTaskTQTy = SavedKmpTaskTQTy;
5028 }
5029 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5030 // Build particular struct kmp_task_t for the given task.
5031 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5032 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5033 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5034 QualType KmpTaskTWithPrivatesPtrQTy =
5035 C.getPointerType(KmpTaskTWithPrivatesQTy);
5036 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5037 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5038 KmpTaskTWithPrivatesTy->getPointerTo();
5039 llvm::Value *KmpTaskTWithPrivatesTySize =
5040 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5041 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5042
5043 // Emit initial values for private copies (if any).
5044 llvm::Value *TaskPrivatesMap = nullptr;
5045 llvm::Type *TaskPrivatesMapTy =
5046 std::next(TaskFunction->arg_begin(), 3)->getType();
5047 if (!Privates.empty()) {
5048 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5049 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5050 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5051 FI->getType(), Privates);
5052 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5053 TaskPrivatesMap, TaskPrivatesMapTy);
5054 } else {
5055 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5056 cast<llvm::PointerType>(TaskPrivatesMapTy));
5057 }
5058 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5059 // kmp_task_t *tt);
5060 llvm::Function *TaskEntry = emitProxyTaskFunction(
5061 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5062 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5063 TaskPrivatesMap);
5064
5065 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5066 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5067 // kmp_routine_entry_t *task_entry);
5068 // Task flags. Format is taken from
5069 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5070 // description of kmp_tasking_flags struct.
5071 enum {
5072 TiedFlag = 0x1,
5073 FinalFlag = 0x2,
5074 DestructorsFlag = 0x8,
5075 PriorityFlag = 0x20
5076 };
5077 unsigned Flags = Data.Tied ? TiedFlag : 0;
5078 bool NeedsCleanup = false;
5079 if (!Privates.empty()) {
5080 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5081 if (NeedsCleanup)
5082 Flags = Flags | DestructorsFlag;
5083 }
5084 if (Data.Priority.getInt())
5085 Flags = Flags | PriorityFlag;
5086 llvm::Value *TaskFlags =
5087 Data.Final.getPointer()
5088 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5089 CGF.Builder.getInt32(FinalFlag),
5090 CGF.Builder.getInt32(/*C=*/0))
5091 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5092 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5093 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5094 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5095 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5096 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5097 TaskEntry, KmpRoutineEntryPtrTy)};
5098 llvm::Value *NewTask;
5099 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5100 // Check if we have any device clause associated with the directive.
5101 const Expr *Device = nullptr;
5102 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5103 Device = C->getDevice();
5104 // Emit device ID if any otherwise use default value.
5105 llvm::Value *DeviceID;
5106 if (Device)
5107 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5108 CGF.Int64Ty, /*isSigned=*/true);
5109 else
5110 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5111 AllocArgs.push_back(DeviceID);
5112 NewTask = CGF.EmitRuntimeCall(
5113 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5114 } else {
5115 NewTask = CGF.EmitRuntimeCall(
5116 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5117 }
5118 llvm::Value *NewTaskNewTaskTTy =
5119 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5120 NewTask, KmpTaskTWithPrivatesPtrTy);
5121 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5122 KmpTaskTWithPrivatesQTy);
5123 LValue TDBase =
5124 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5125 // Fill the data in the resulting kmp_task_t record.
5126 // Copy shareds if there are any.
5127 Address KmpTaskSharedsPtr = Address::invalid();
5128 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5129 KmpTaskSharedsPtr =
5130 Address(CGF.EmitLoadOfScalar(
5131 CGF.EmitLValueForField(
5132 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5133 KmpTaskTShareds)),
5134 Loc),
5135 CGF.getNaturalTypeAlignment(SharedsTy));
5136 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5137 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5138 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5139 }
5140 // Emit initial values for private copies (if any).
5141 TaskResultTy Result;
5142 if (!Privates.empty()) {
5143 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5144 SharedsTy, SharedsPtrTy, Data, Privates,
5145 /*ForDup=*/false);
5146 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5147 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5148 Result.TaskDupFn = emitTaskDupFunction(
5149 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5150 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5151 /*WithLastIter=*/!Data.LastprivateVars.empty());
5152 }
5153 }
5154 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5155 enum { Priority = 0, Destructors = 1 };
5156 // Provide pointer to function with destructors for privates.
5157 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5158 const RecordDecl *KmpCmplrdataUD =
5159 (*FI)->getType()->getAsUnionType()->getDecl();
5160 if (NeedsCleanup) {
5161 llvm::Value *DestructorFn = emitDestructorsFunction(
5162 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5163 KmpTaskTWithPrivatesQTy);
5164 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5165 LValue DestructorsLV = CGF.EmitLValueForField(
5166 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5167 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5168 DestructorFn, KmpRoutineEntryPtrTy),
5169 DestructorsLV);
5170 }
5171 // Set priority.
5172 if (Data.Priority.getInt()) {
5173 LValue Data2LV = CGF.EmitLValueForField(
5174 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5175 LValue PriorityLV = CGF.EmitLValueForField(
5176 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5177 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5178 }
5179 Result.NewTask = NewTask;
5180 Result.TaskEntry = TaskEntry;
5181 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5182 Result.TDBase = TDBase;
5183 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5184 return Result;
5185}
5186
5187void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5188 const OMPExecutableDirective &D,
5189 llvm::Function *TaskFunction,
5190 QualType SharedsTy, Address Shareds,
5191 const Expr *IfCond,
5192 const OMPTaskDataTy &Data) {
5193 if (!CGF.HaveInsertPoint())
5194 return;
5195
5196 TaskResultTy Result =
5197 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5198 llvm::Value *NewTask = Result.NewTask;
5199 llvm::Function *TaskEntry = Result.TaskEntry;
5200 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5201 LValue TDBase = Result.TDBase;
5202 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5203 ASTContext &C = CGM.getContext();
5204 // Process list of dependences.
5205 Address DependenciesArray = Address::invalid();
5206 unsigned NumDependencies = Data.Dependences.size();
5207 if (NumDependencies) {
5208 // Dependence kind for RTL.
5209 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5210 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5211 RecordDecl *KmpDependInfoRD;
5212 QualType FlagsTy =
5213 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5214 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5215 if (KmpDependInfoTy.isNull()) {
5216 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5217 KmpDependInfoRD->startDefinition();
5218 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5219 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5220 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5221 KmpDependInfoRD->completeDefinition();
5222 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5223 } else {
5224 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5225 }
5226 // Define type kmp_depend_info[<Dependences.size()>];
5227 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5228 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5229 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
5230 // kmp_depend_info[<Dependences.size()>] deps;
5231 DependenciesArray =
5232 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5233 for (unsigned I = 0; I < NumDependencies; ++I) {
5234 const Expr *E = Data.Dependences[I].second;
5235 LValue Addr = CGF.EmitLValue(E);
5236 llvm::Value *Size;
5237 QualType Ty = E->getType();
5238 if (const auto *ASE =
5239 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5240 LValue UpAddrLVal =
5241 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5242 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
5243 UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
5244 llvm::Value *LowIntPtr =
5245 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy);
5246 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5247 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5248 } else {
5249 Size = CGF.getTypeSize(Ty);
5250 }
5251 LValue Base = CGF.MakeAddrLValue(
5252 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5253 KmpDependInfoTy);
5254 // deps[i].base_addr = &<Dependences[i].second>;
5255 LValue BaseAddrLVal = CGF.EmitLValueForField(
5256 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5257 CGF.EmitStoreOfScalar(
5258 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy),
5259 BaseAddrLVal);
5260 // deps[i].len = sizeof(<Dependences[i].second>);
5261 LValue LenLVal = CGF.EmitLValueForField(
5262 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5263 CGF.EmitStoreOfScalar(Size, LenLVal);
5264 // deps[i].flags = <Dependences[i].first>;
5265 RTLDependenceKindTy DepKind;
5266 switch (Data.Dependences[I].first) {
5267 case OMPC_DEPEND_in:
5268 DepKind = DepIn;
5269 break;
5270 // Out and InOut dependencies must use the same code.
5271 case OMPC_DEPEND_out:
5272 case OMPC_DEPEND_inout:
5273 DepKind = DepInOut;
5274 break;
5275 case OMPC_DEPEND_mutexinoutset:
5276 DepKind = DepMutexInOutSet;
5277 break;
5278 case OMPC_DEPEND_source:
5279 case OMPC_DEPEND_sink:
5280 case OMPC_DEPEND_unknown:
5281 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-11~++20200226111113+80d7e473e0b/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5281)
;
5282 }
5283 LValue FlagsLVal = CGF.EmitLValueForField(
5284 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5285 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5286 FlagsLVal);
5287 }
5288 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5289 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5290 }
5291
5292 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5293 // libcall.
5294 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5295 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5296 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5297 // list is not empty
5298 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5299 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5300 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5301 llvm::Value *DepTaskArgs[7];
5302 if (NumDependencies) {
5303 DepTaskArgs[0] = UpLoc;
5304 DepTaskArgs[1] = ThreadID;
5305 DepTaskArgs[2] = NewTask;
5306 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5307 DepTaskArgs[4] = DependenciesArray.getPointer();
5308 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5309 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5310 }
5311 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5312 &TaskArgs,
5313 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5314 if (!Data.Tied) {
5315 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5316 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5317 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5318 }
5319 if (NumDependencies) {
5320 CGF.EmitRuntimeCall(
5321 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5322 } else {
5323 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5324 TaskArgs);
5325 }
5326 // Check if parent region is untied and build return for untied task;
5327 if (auto *Region =
5328 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5329 Region->emitUntiedSwitch(CGF);
5330 };
5331
5332 llvm::Value *DepWaitTaskArgs[6];
5333 if (NumDependencies) {
5334 DepWaitTaskArgs[0] = UpLoc;
5335 DepWaitTaskArgs[1] = ThreadID;
5336 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5337 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5338 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5339 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5340 }
5341 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5342 NumDependencies, &DepWaitTaskArgs,
5343 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5344 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5345 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5346 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5347 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5348 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5349 // is specified.
5350 if (NumDependencies)
5351 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5352 DepWaitTaskArgs);
5353 // Call proxy_task_entry(gtid, new_task);
5354 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5355 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5356 Action.Enter(CGF);
5357 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5358 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5359 OutlinedFnArgs);
5360 };
5361
5362 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5363 // kmp_task_t *new_task);
5364 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5365 // kmp_task_t *new_task);
5366 RegionCodeGenTy RCG(CodeGen);
5367 CommonActionTy Action(
5368 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5369 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5370 RCG.setAction(Action);
5371 RCG(CGF);
5372 };
5373
5374 if (IfCond) {
5375 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5376 } else {
5377 RegionCodeGenTy ThenRCG(ThenCodeGen);
5378 ThenRCG(CGF);
5379 }
5380}
5381
5382void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5383 const OMPLoopDirective &D,
5384 llvm::Function *TaskFunction,
5385 QualType SharedsTy, Address Shareds,
5386 const Expr *IfCond,
5387 const OMPTaskDataTy &Data) {
5388 if (!CGF.HaveInsertPoint())
5389 return;
5390 TaskResultTy Result =
5391 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5392 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5393 // libcall.
5394 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5395 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5396 // sched, kmp_uint64 grainsize, void *task_dup);
5397 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5398 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5399 llvm::Value *IfVal;
5400 if (IfCond) {
5401 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5402 /*isSigned=*/true);
5403 } else {
5404 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5405 }
5406
5407 LValue LBLVal = CGF.EmitLValueForField(
5408 Result.TDBase,
5409 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5410 const auto *LBVar =
5411 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5412 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5413 LBLVal.getQuals(),
5414 /*IsInitializer=*/true);
5415 LValue UBLVal = CGF.EmitLValueForField(
5416 Result.TDBase,
5417 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5418 const auto *UBVar =
5419 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5420 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5421 UBLVal.getQuals(),
5422 /*IsInitializer=*/true);
5423 LValue StLVal = CGF.EmitLValueForField(
5424 Result.TDBase,
5425 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5426 const auto *StVar =
5427 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5428 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5429 StLVal.getQuals(),
5430 /*IsInitializer=*/true);
5431 // Store reductions address.
5432 LValue RedLVal = CGF.EmitLValueForField(
5433 Result.TDBase,
5434 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5435 if (Data.Reductions) {
5436 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5437 } else {
5438 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5439 CGF.getContext().VoidPtrTy);
5440 }
5441 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5442 llvm::Value *TaskArgs[] = {
5443 UpLoc,
5444 ThreadID,
5445 Result.NewTask,
5446 IfVal,
5447 LBLVal.getPointer(CGF),
5448 UBLVal.getPointer(CGF),
5449 CGF.EmitLoadOfScalar(StLVal, Loc),
5450 llvm::ConstantInt::getSigned(
5451 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5452 llvm::ConstantInt::getSigned(
5453 CGF.IntTy, Data.Schedule.getPointer()
5454 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5455 : NoSchedule),
5456 Data.Schedule.getPointer()
5457 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5458 /*isSigned=*/false)
5459 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5460 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5461 Result.TaskDupFn, CGF.VoidPtrTy)
5462 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5463 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5464}
5465
5466/// Emit reduction operation for each element of array (required for
5467/// array sections) LHS op = RHS.
5468/// \param Type Type of array.
5469/// \param LHSVar Variable on the left side of the reduction operation
5470/// (references element of array in original variable).
5471/// \param RHSVar Variable on the right side of the reduction operation
5472/// (references element of array in original variable).
5473/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5474/// RHSVar.
5475static void EmitOMPAggregateReduction(
5476 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5477 const VarDecl *RHSVar,
5478 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5479 const Expr *, const Expr *)> &RedOpGen,
5480 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5481 const Expr *UpExpr = nullptr) {
5482 // Perform element-by-element initialization.
5483 QualType ElementTy;
5484 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5485 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5486
5487 // Drill down to the base element type on both arrays.
5488 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5489 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5490
5491 llvm::Value *RHSBegin = RHSAddr.getPointer();
5492 llvm::Value *LHSBegin = LHSAddr.getPointer();
5493 // Cast from pointer to array type to pointer to single element.
5494 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5495 // The basic structure here is a while-do loop.
5496 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5497 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5498 llvm::Value *IsEmpty =
5499 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5500 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5501
5502 // Enter the loop body, making that address the current address.
5503 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5504 CGF.EmitBlock(BodyBB);
5505
5506 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5507
5508 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5509 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5510 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5511 Address RHSElementCurrent =
5512 Address(RHSElementPHI,
5513 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5514
5515 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5516 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5517 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5518 Address LHSElementCurrent =
5519 Address(LHSElementPHI,
5520 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5521
5522 // Emit copy.
5523 CodeGenFunction::OMPPrivateScope Scope(CGF);
5524 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5525 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5526 Scope.Privatize();
5527 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5528 Scope.ForceCleanup();
5529
5530 // Shift the address forward by one element.
5531 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5532 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5533 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5534 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5535 // Check whether we've reached the end.
5536 llvm::Value *Done =
5537 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5538 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5539 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5540 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5541
5542 // Done.
5543 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5544}
5545
5546/// Emit reduction combiner. If the combiner is a simple expression emit it as
5547/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5548/// UDR combiner function.
5549static void emitReductionCombiner(CodeGenFunction &CGF,
5550 const Expr *ReductionOp) {
5551 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5552 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5553 if (const auto *DRE =
5554 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5555 if (const auto *DRD =
5556 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5557 std::pair<llvm::Function *, llvm::Function *> Reduction =
5558 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5559 RValue Func = RValue::get(Reduction.first);
5560 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5561 CGF.EmitIgnoredExpr(ReductionOp);
5562 return;
5563 }
5564 CGF.EmitIgnoredExpr(ReductionOp);
5565}
5566
5567llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5568 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5569 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5570 ArrayRef<const Expr *> ReductionOps) {
5571 ASTContext &C = CGM.getContext();
5572
5573 // void reduction_func(void *LHSArg, void *RHSArg);
5574 FunctionArgList Args;
5575 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5576 ImplicitParamDecl::Other);
5577 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5578 ImplicitParamDecl::Other);
5579 Args.push_back(&LHSArg);
5580 Args.push_back(&RHSArg);
5581 const auto &CGFI =
5582 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5583 std::string Name = getName({"omp", "reduction", "reduction_func"});
5584 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5585 llvm::GlobalValue::InternalLinkage, Name,
5586 &CGM.getModule());
5587 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5588 Fn->setDoesNotRecurse();
5589 CodeGenFunction CGF(CGM);
5590 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5591
5592 // Dst = (void*[n])(LHSArg);
5593 // Src = (void*[n])(RHSArg);
5594 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5595 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5596 ArgsType), CGF.getPointerAlign());
5597 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5598 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5599 ArgsType), CGF.getPointerAlign());
5600
5601 // ...
5602 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5603 // ...
5604 CodeGenFunction::OMPPrivateScope Scope(CGF);
5605 auto IPriv = Privates.begin();
5606 unsigned Idx = 0;
5607 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5608 const auto *RHSVar =
5609 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5610 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5611 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5612 });
5613 const auto *LHSVar =
5614 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5615 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5616 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5617 });
5618 QualType PrivTy = (*IPriv)->getType();
5619 if (PrivTy->isVariablyModifiedType()) {
5620 // Get array size and emit VLA type.
5621 ++Idx;
5622 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5623 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5624 const VariableArrayType *VLA =
5625 CGF.getContext().getAsVariableArrayType(PrivTy);
5626 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5627 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5628 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5629 CGF.EmitVariablyModifiedType(PrivTy);
5630 }
5631 }
5632 Scope.Privatize();
5633 IPriv = Privates.begin();
5634 auto ILHS = LHSExprs.begin();
5635 auto IRHS = RHSExprs.begin();
5636 for (const Expr *E : ReductionOps) {
5637 if ((*IPriv)->getType()->isArrayType()) {
5638 // Emit reduction for array section.
5639 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5640 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5641 EmitOMPAggregateReduction(
5642 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5643 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5644 emitReductionCombiner(CGF, E);
5645 });
5646 } else {
5647 // Emit reduction for array subscript or single variable.
5648 emitReductionCombiner(CGF, E);
5649 }
5650 ++IPriv;
5651 ++ILHS;
5652 ++IRHS;
5653 }
5654 Scope.ForceCleanup();
5655 CGF.FinishFunction();
5656 return Fn;
5657}
5658
5659void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5660 const Expr *ReductionOp,
5661 const Expr *PrivateRef,
5662 const DeclRefExpr *LHS,
5663 const DeclRefExpr *RHS) {
5664 if (PrivateRef->getType()->isArrayType()) {
5665 // Emit reduction for array section.
5666 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5667 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5668 EmitOMPAggregateReduction(
5669 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5670 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5671 emitReductionCombiner(CGF, ReductionOp);
5672 });
5673 } else {
5674 // Emit reduction for array subscript or single variable.
5675 emitReductionCombiner(CGF, ReductionOp);
5676 }
5677}
5678
5679void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5680 ArrayRef<const Expr *> Privates,
5681 ArrayRef<const Expr *> LHSExprs,
5682 ArrayRef<const Expr *> RHSExprs,
5683 ArrayRef<const Expr *> ReductionOps,
5684 ReductionOptionsTy Options) {
5685 if (!CGF.HaveInsertPoint())
5686 return;
5687
5688 bool WithNowait = Options.WithNowait;
5689 bool SimpleReduction = Options.SimpleReduction;
5690
5691 // Next code should be emitted for reduction:
5692 //
5693 // static kmp_critical_name lock = { 0 };
5694 //
5695 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5696 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5697 // ...
5698 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5699 // *(Type<n>-1*)rhs[<n>-1]);
5700 // }
5701 //
5702 // ...
5703 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5704 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5705 // RedList, reduce_func, &<lock>)) {
5706 // case 1:
5707 // ...
5708 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5709 // ...
5710 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5711 // break;
5712 // case 2:
5713 // ...
5714 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5715 // ...
5716 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5717 // break;
5718 // default:;
5719 // }
5720 //
5721 // if SimpleReduction is true, only the next code is generated:
5722 // ...
5723 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5724 // ...
5725
5726 ASTContext &C = CGM.getContext();
5727
5728 if (SimpleReduction) {
5729 CodeGenFunction::RunCleanupsScope Scop