Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8336, column 15
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-13-084841-49055-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/CodeGen/ConstantInitBuilder.h"
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/SetOperations.h"
27#include "llvm/Bitcode/BitcodeReader.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/Value.h"
32#include "llvm/Support/Format.h"
33#include "llvm/Support/raw_ostream.h"
34#include <cassert>
35
36using namespace clang;
37using namespace CodeGen;
38using namespace llvm::omp;
39
40namespace {
41/// Base class for handling code generation inside OpenMP regions.
42class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
43public:
44 /// Kinds of OpenMP regions used in codegen.
45 enum CGOpenMPRegionKind {
46 /// Region with outlined function for standalone 'parallel'
47 /// directive.
48 ParallelOutlinedRegion,
49 /// Region with outlined function for standalone 'task' directive.
50 TaskOutlinedRegion,
51 /// Region for constructs that do not require function outlining,
52 /// like 'for', 'sections', 'atomic' etc. directives.
53 InlinedRegion,
54 /// Region with outlined function for standalone 'target' directive.
55 TargetRegion,
56 };
57
58 CGOpenMPRegionInfo(const CapturedStmt &CS,
59 const CGOpenMPRegionKind RegionKind,
60 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61 bool HasCancel)
62 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
63 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
64
65 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
66 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
67 bool HasCancel)
68 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
69 Kind(Kind), HasCancel(HasCancel) {}
70
71 /// Get a variable or parameter for storing global thread id
72 /// inside OpenMP construct.
73 virtual const VarDecl *getThreadIDVariable() const = 0;
74
75 /// Emit the captured statement body.
76 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
77
78 /// Get an LValue for the current ThreadID variable.
79 /// \return LValue for thread id variable. This LValue always has type int32*.
80 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
81
82 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
83
84 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
85
86 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
87
88 bool hasCancel() const { return HasCancel; }
89
90 static bool classof(const CGCapturedStmtInfo *Info) {
91 return Info->getKind() == CR_OpenMP;
92 }
93
94 ~CGOpenMPRegionInfo() override = default;
95
96protected:
97 CGOpenMPRegionKind RegionKind;
98 RegionCodeGenTy CodeGen;
99 OpenMPDirectiveKind Kind;
100 bool HasCancel;
101};
102
103/// API for captured statement code generation in OpenMP constructs.
104class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
105public:
106 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
107 const RegionCodeGenTy &CodeGen,
108 OpenMPDirectiveKind Kind, bool HasCancel,
109 StringRef HelperName)
110 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
111 HasCancel),
112 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
113 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 113, __PRETTY_FUNCTION__))
;
114 }
115
116 /// Get a variable or parameter for storing global thread id
117 /// inside OpenMP construct.
118 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
119
120 /// Get the name of the capture helper.
121 StringRef getHelperName() const override { return HelperName; }
122
123 static bool classof(const CGCapturedStmtInfo *Info) {
124 return CGOpenMPRegionInfo::classof(Info) &&
125 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
126 ParallelOutlinedRegion;
127 }
128
129private:
130 /// A variable or parameter storing global thread id for OpenMP
131 /// constructs.
132 const VarDecl *ThreadIDVar;
133 StringRef HelperName;
134};
135
136/// API for captured statement code generation in OpenMP constructs.
137class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
138public:
139 class UntiedTaskActionTy final : public PrePostActionTy {
140 bool Untied;
141 const VarDecl *PartIDVar;
142 const RegionCodeGenTy UntiedCodeGen;
143 llvm::SwitchInst *UntiedSwitch = nullptr;
144
145 public:
146 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
147 const RegionCodeGenTy &UntiedCodeGen)
148 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
149 void Enter(CodeGenFunction &CGF) override {
150 if (Untied) {
151 // Emit task switching point.
152 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
153 CGF.GetAddrOfLocalVar(PartIDVar),
154 PartIDVar->getType()->castAs<PointerType>());
155 llvm::Value *Res =
156 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
157 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
158 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
159 CGF.EmitBlock(DoneBB);
160 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
161 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
162 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
163 CGF.Builder.GetInsertBlock());
164 emitUntiedSwitch(CGF);
165 }
166 }
167 void emitUntiedSwitch(CodeGenFunction &CGF) const {
168 if (Untied) {
169 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
170 CGF.GetAddrOfLocalVar(PartIDVar),
171 PartIDVar->getType()->castAs<PointerType>());
172 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
173 PartIdLVal);
174 UntiedCodeGen(CGF);
175 CodeGenFunction::JumpDest CurPoint =
176 CGF.getJumpDestInCurrentScope(".untied.next.");
177 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
178 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
179 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
180 CGF.Builder.GetInsertBlock());
181 CGF.EmitBranchThroughCleanup(CurPoint);
182 CGF.EmitBlock(CurPoint.getBlock());
183 }
184 }
185 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
186 };
187 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
188 const VarDecl *ThreadIDVar,
189 const RegionCodeGenTy &CodeGen,
190 OpenMPDirectiveKind Kind, bool HasCancel,
191 const UntiedTaskActionTy &Action)
192 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
193 ThreadIDVar(ThreadIDVar), Action(Action) {
194 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 194, __PRETTY_FUNCTION__))
;
195 }
196
197 /// Get a variable or parameter for storing global thread id
198 /// inside OpenMP construct.
199 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
200
201 /// Get an LValue for the current ThreadID variable.
202 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
203
204 /// Get the name of the capture helper.
205 StringRef getHelperName() const override { return ".omp_outlined."; }
206
207 void emitUntiedSwitch(CodeGenFunction &CGF) override {
208 Action.emitUntiedSwitch(CGF);
209 }
210
211 static bool classof(const CGCapturedStmtInfo *Info) {
212 return CGOpenMPRegionInfo::classof(Info) &&
213 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
214 TaskOutlinedRegion;
215 }
216
217private:
218 /// A variable or parameter storing global thread id for OpenMP
219 /// constructs.
220 const VarDecl *ThreadIDVar;
221 /// Action for emitting code for untied tasks.
222 const UntiedTaskActionTy &Action;
223};
224
225/// API for inlined captured statement code generation in OpenMP
226/// constructs.
227class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
228public:
229 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
230 const RegionCodeGenTy &CodeGen,
231 OpenMPDirectiveKind Kind, bool HasCancel)
232 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
233 OldCSI(OldCSI),
234 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
235
236 // Retrieve the value of the context parameter.
237 llvm::Value *getContextValue() const override {
238 if (OuterRegionInfo)
239 return OuterRegionInfo->getContextValue();
240 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 240)
;
241 }
242
243 void setContextValue(llvm::Value *V) override {
244 if (OuterRegionInfo) {
245 OuterRegionInfo->setContextValue(V);
246 return;
247 }
248 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 248)
;
249 }
250
251 /// Lookup the captured field decl for a variable.
252 const FieldDecl *lookup(const VarDecl *VD) const override {
253 if (OuterRegionInfo)
254 return OuterRegionInfo->lookup(VD);
255 // If there is no outer outlined region,no need to lookup in a list of
256 // captured variables, we can use the original one.
257 return nullptr;
258 }
259
260 FieldDecl *getThisFieldDecl() const override {
261 if (OuterRegionInfo)
262 return OuterRegionInfo->getThisFieldDecl();
263 return nullptr;
264 }
265
266 /// Get a variable or parameter for storing global thread id
267 /// inside OpenMP construct.
268 const VarDecl *getThreadIDVariable() const override {
269 if (OuterRegionInfo)
270 return OuterRegionInfo->getThreadIDVariable();
271 return nullptr;
272 }
273
274 /// Get an LValue for the current ThreadID variable.
275 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
276 if (OuterRegionInfo)
277 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
278 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 278)
;
279 }
280
281 /// Get the name of the capture helper.
282 StringRef getHelperName() const override {
283 if (auto *OuterRegionInfo = getOldCSI())
284 return OuterRegionInfo->getHelperName();
285 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 285)
;
286 }
287
288 void emitUntiedSwitch(CodeGenFunction &CGF) override {
289 if (OuterRegionInfo)
290 OuterRegionInfo->emitUntiedSwitch(CGF);
291 }
292
293 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
294
295 static bool classof(const CGCapturedStmtInfo *Info) {
296 return CGOpenMPRegionInfo::classof(Info) &&
297 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
298 }
299
300 ~CGOpenMPInlinedRegionInfo() override = default;
301
302private:
303 /// CodeGen info about outer OpenMP region.
304 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
305 CGOpenMPRegionInfo *OuterRegionInfo;
306};
307
308/// API for captured statement code generation in OpenMP target
309/// constructs. For this captures, implicit parameters are used instead of the
310/// captured fields. The name of the target region has to be unique in a given
311/// application so it is provided by the client, because only the client has
312/// the information to generate that.
313class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
314public:
315 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
316 const RegionCodeGenTy &CodeGen, StringRef HelperName)
317 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
318 /*HasCancel=*/false),
319 HelperName(HelperName) {}
320
321 /// This is unused for target regions because each starts executing
322 /// with a single thread.
323 const VarDecl *getThreadIDVariable() const override { return nullptr; }
324
325 /// Get the name of the capture helper.
326 StringRef getHelperName() const override { return HelperName; }
327
328 static bool classof(const CGCapturedStmtInfo *Info) {
329 return CGOpenMPRegionInfo::classof(Info) &&
330 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
331 }
332
333private:
334 StringRef HelperName;
335};
336
337static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
338 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 338)
;
339}
340/// API for generation of expressions captured in a innermost OpenMP
341/// region.
342class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
343public:
344 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
345 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
346 OMPD_unknown,
347 /*HasCancel=*/false),
348 PrivScope(CGF) {
349 // Make sure the globals captured in the provided statement are local by
350 // using the privatization logic. We assume the same variable is not
351 // captured more than once.
352 for (const auto &C : CS.captures()) {
353 if (!C.capturesVariable() && !C.capturesVariableByCopy())
354 continue;
355
356 const VarDecl *VD = C.getCapturedVar();
357 if (VD->isLocalVarDeclOrParm())
358 continue;
359
360 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
361 /*RefersToEnclosingVariableOrCapture=*/false,
362 VD->getType().getNonReferenceType(), VK_LValue,
363 C.getLocation());
364 PrivScope.addPrivate(
365 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
366 }
367 (void)PrivScope.Privatize();
368 }
369
370 /// Lookup the captured field decl for a variable.
371 const FieldDecl *lookup(const VarDecl *VD) const override {
372 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
373 return FD;
374 return nullptr;
375 }
376
377 /// Emit the captured statement body.
378 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
379 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 379)
;
380 }
381
382 /// Get a variable or parameter for storing global thread id
383 /// inside OpenMP construct.
384 const VarDecl *getThreadIDVariable() const override {
385 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 385)
;
386 }
387
388 /// Get the name of the capture helper.
389 StringRef getHelperName() const override {
390 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 390)
;
391 }
392
393 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
394
395private:
396 /// Private scope to capture global variables.
397 CodeGenFunction::OMPPrivateScope PrivScope;
398};
399
400/// RAII for emitting code of OpenMP constructs.
401class InlinedOpenMPRegionRAII {
402 CodeGenFunction &CGF;
403 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
404 FieldDecl *LambdaThisCaptureField = nullptr;
405 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
406
407public:
408 /// Constructs region for combined constructs.
409 /// \param CodeGen Code generation sequence for combined directives. Includes
410 /// a list of functions used for code generation of implicitly inlined
411 /// regions.
412 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
413 OpenMPDirectiveKind Kind, bool HasCancel)
414 : CGF(CGF) {
415 // Start emission for the construct.
416 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
417 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
418 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
419 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
420 CGF.LambdaThisCaptureField = nullptr;
421 BlockInfo = CGF.BlockInfo;
422 CGF.BlockInfo = nullptr;
423 }
424
425 ~InlinedOpenMPRegionRAII() {
426 // Restore original CapturedStmtInfo only if we're done with code emission.
427 auto *OldCSI =
428 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
429 delete CGF.CapturedStmtInfo;
430 CGF.CapturedStmtInfo = OldCSI;
431 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
432 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
433 CGF.BlockInfo = BlockInfo;
434 }
435};
436
437/// Values for bit flags used in the ident_t to describe the fields.
438/// All enumeric elements are named and described in accordance with the code
439/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
440enum OpenMPLocationFlags : unsigned {
441 /// Use trampoline for internal microtask.
442 OMP_IDENT_IMD = 0x01,
443 /// Use c-style ident structure.
444 OMP_IDENT_KMPC = 0x02,
445 /// Atomic reduction option for kmpc_reduce.
446 OMP_ATOMIC_REDUCE = 0x10,
447 /// Explicit 'barrier' directive.
448 OMP_IDENT_BARRIER_EXPL = 0x20,
449 /// Implicit barrier in code.
450 OMP_IDENT_BARRIER_IMPL = 0x40,
451 /// Implicit barrier in 'for' directive.
452 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
453 /// Implicit barrier in 'sections' directive.
454 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
455 /// Implicit barrier in 'single' directive.
456 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
457 /// Call of __kmp_for_static_init for static loop.
458 OMP_IDENT_WORK_LOOP = 0x200,
459 /// Call of __kmp_for_static_init for sections.
460 OMP_IDENT_WORK_SECTIONS = 0x400,
461 /// Call of __kmp_for_static_init for distribute.
462 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
463 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
464};
465
466namespace {
467LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
468/// Values for bit flags for marking which requires clauses have been used.
469enum OpenMPOffloadingRequiresDirFlags : int64_t {
470 /// flag undefined.
471 OMP_REQ_UNDEFINED = 0x000,
472 /// no requires clause present.
473 OMP_REQ_NONE = 0x001,
474 /// reverse_offload clause.
475 OMP_REQ_REVERSE_OFFLOAD = 0x002,
476 /// unified_address clause.
477 OMP_REQ_UNIFIED_ADDRESS = 0x004,
478 /// unified_shared_memory clause.
479 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
480 /// dynamic_allocators clause.
481 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
482 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
483};
484
485enum OpenMPOffloadingReservedDeviceIDs {
486 /// Device ID if the device was not defined, runtime should get it
487 /// from environment variables in the spec.
488 OMP_DEVICEID_UNDEF = -1,
489};
490} // anonymous namespace
491
492/// Describes ident structure that describes a source location.
493/// All descriptions are taken from
494/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
495/// Original structure:
496/// typedef struct ident {
497/// kmp_int32 reserved_1; /**< might be used in Fortran;
498/// see above */
499/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
500/// KMP_IDENT_KMPC identifies this union
501/// member */
502/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
503/// see above */
504///#if USE_ITT_BUILD
505/// /* but currently used for storing
506/// region-specific ITT */
507/// /* contextual information. */
508///#endif /* USE_ITT_BUILD */
509/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
510/// C++ */
511/// char const *psource; /**< String describing the source location.
512/// The string is composed of semi-colon separated
513// fields which describe the source file,
514/// the function and a pair of line numbers that
515/// delimit the construct.
516/// */
517/// } ident_t;
518enum IdentFieldIndex {
519 /// might be used in Fortran
520 IdentField_Reserved_1,
521 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
522 IdentField_Flags,
523 /// Not really used in Fortran any more
524 IdentField_Reserved_2,
525 /// Source[4] in Fortran, do not use for C++
526 IdentField_Reserved_3,
527 /// String describing the source location. The string is composed of
528 /// semi-colon separated fields which describe the source file, the function
529 /// and a pair of line numbers that delimit the construct.
530 IdentField_PSource
531};
532
533/// Schedule types for 'omp for' loops (these enumerators are taken from
534/// the enum sched_type in kmp.h).
535enum OpenMPSchedType {
536 /// Lower bound for default (unordered) versions.
537 OMP_sch_lower = 32,
538 OMP_sch_static_chunked = 33,
539 OMP_sch_static = 34,
540 OMP_sch_dynamic_chunked = 35,
541 OMP_sch_guided_chunked = 36,
542 OMP_sch_runtime = 37,
543 OMP_sch_auto = 38,
544 /// static with chunk adjustment (e.g., simd)
545 OMP_sch_static_balanced_chunked = 45,
546 /// Lower bound for 'ordered' versions.
547 OMP_ord_lower = 64,
548 OMP_ord_static_chunked = 65,
549 OMP_ord_static = 66,
550 OMP_ord_dynamic_chunked = 67,
551 OMP_ord_guided_chunked = 68,
552 OMP_ord_runtime = 69,
553 OMP_ord_auto = 70,
554 OMP_sch_default = OMP_sch_static,
555 /// dist_schedule types
556 OMP_dist_sch_static_chunked = 91,
557 OMP_dist_sch_static = 92,
558 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
559 /// Set if the monotonic schedule modifier was present.
560 OMP_sch_modifier_monotonic = (1 << 29),
561 /// Set if the nonmonotonic schedule modifier was present.
562 OMP_sch_modifier_nonmonotonic = (1 << 30),
563};
564
565enum OpenMPRTLFunction {
566 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
567 /// kmpc_micro microtask, ...);
568 OMPRTL__kmpc_fork_call,
569 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
570 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
571 OMPRTL__kmpc_threadprivate_cached,
572 /// Call to void __kmpc_threadprivate_register( ident_t *,
573 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
574 OMPRTL__kmpc_threadprivate_register,
575 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
576 OMPRTL__kmpc_global_thread_num,
577 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
578 // kmp_critical_name *crit);
579 OMPRTL__kmpc_critical,
580 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
581 // global_tid, kmp_critical_name *crit, uintptr_t hint);
582 OMPRTL__kmpc_critical_with_hint,
583 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
584 // kmp_critical_name *crit);
585 OMPRTL__kmpc_end_critical,
586 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
587 // global_tid);
588 OMPRTL__kmpc_cancel_barrier,
589 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
590 OMPRTL__kmpc_barrier,
591 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
592 OMPRTL__kmpc_for_static_fini,
593 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
594 // global_tid);
595 OMPRTL__kmpc_serialized_parallel,
596 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
597 // global_tid);
598 OMPRTL__kmpc_end_serialized_parallel,
599 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
600 // kmp_int32 num_threads);
601 OMPRTL__kmpc_push_num_threads,
602 // Call to void __kmpc_flush(ident_t *loc);
603 OMPRTL__kmpc_flush,
604 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
605 OMPRTL__kmpc_master,
606 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
607 OMPRTL__kmpc_end_master,
608 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
609 // int end_part);
610 OMPRTL__kmpc_omp_taskyield,
611 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
612 OMPRTL__kmpc_single,
613 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
614 OMPRTL__kmpc_end_single,
615 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
616 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
617 // kmp_routine_entry_t *task_entry);
618 OMPRTL__kmpc_omp_task_alloc,
619 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
620 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
621 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
622 // kmp_int64 device_id);
623 OMPRTL__kmpc_omp_target_task_alloc,
624 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
625 // new_task);
626 OMPRTL__kmpc_omp_task,
627 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
628 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
629 // kmp_int32 didit);
630 OMPRTL__kmpc_copyprivate,
631 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
632 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
633 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
634 OMPRTL__kmpc_reduce,
635 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
636 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
637 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
638 // *lck);
639 OMPRTL__kmpc_reduce_nowait,
640 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
641 // kmp_critical_name *lck);
642 OMPRTL__kmpc_end_reduce,
643 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
644 // kmp_critical_name *lck);
645 OMPRTL__kmpc_end_reduce_nowait,
646 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
647 // kmp_task_t * new_task);
648 OMPRTL__kmpc_omp_task_begin_if0,
649 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
650 // kmp_task_t * new_task);
651 OMPRTL__kmpc_omp_task_complete_if0,
652 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
653 OMPRTL__kmpc_ordered,
654 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
655 OMPRTL__kmpc_end_ordered,
656 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
657 // global_tid);
658 OMPRTL__kmpc_omp_taskwait,
659 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
660 OMPRTL__kmpc_taskgroup,
661 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
662 OMPRTL__kmpc_end_taskgroup,
663 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
664 // int proc_bind);
665 OMPRTL__kmpc_push_proc_bind,
666 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
667 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
668 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
669 OMPRTL__kmpc_omp_task_with_deps,
670 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
671 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
672 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
673 OMPRTL__kmpc_omp_wait_deps,
674 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
675 // global_tid, kmp_int32 cncl_kind);
676 OMPRTL__kmpc_cancellationpoint,
677 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
678 // kmp_int32 cncl_kind);
679 OMPRTL__kmpc_cancel,
680 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
681 // kmp_int32 num_teams, kmp_int32 thread_limit);
682 OMPRTL__kmpc_push_num_teams,
683 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
684 // microtask, ...);
685 OMPRTL__kmpc_fork_teams,
686 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
687 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
688 // sched, kmp_uint64 grainsize, void *task_dup);
689 OMPRTL__kmpc_taskloop,
690 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
691 // num_dims, struct kmp_dim *dims);
692 OMPRTL__kmpc_doacross_init,
693 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
694 OMPRTL__kmpc_doacross_fini,
695 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
696 // *vec);
697 OMPRTL__kmpc_doacross_post,
698 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
699 // *vec);
700 OMPRTL__kmpc_doacross_wait,
701 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
702 // *data);
703 OMPRTL__kmpc_task_reduction_init,
704 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
705 // *d);
706 OMPRTL__kmpc_task_reduction_get_th_data,
707 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
708 OMPRTL__kmpc_alloc,
709 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
710 OMPRTL__kmpc_free,
711
712 //
713 // Offloading related calls
714 //
715 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
716 // size);
717 OMPRTL__kmpc_push_target_tripcount,
718 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
719 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
720 // *arg_types);
721 OMPRTL__tgt_target,
722 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
723 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
724 // *arg_types);
725 OMPRTL__tgt_target_nowait,
726 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
727 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
728 // *arg_types, int32_t num_teams, int32_t thread_limit);
729 OMPRTL__tgt_target_teams,
730 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
731 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
732 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
733 OMPRTL__tgt_target_teams_nowait,
734 // Call to void __tgt_register_requires(int64_t flags);
735 OMPRTL__tgt_register_requires,
736 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
737 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
738 OMPRTL__tgt_target_data_begin,
739 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
740 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
741 // *arg_types);
742 OMPRTL__tgt_target_data_begin_nowait,
743 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
744 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
745 OMPRTL__tgt_target_data_end,
746 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
747 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
748 // *arg_types);
749 OMPRTL__tgt_target_data_end_nowait,
750 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
751 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
752 OMPRTL__tgt_target_data_update,
753 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
754 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
755 // *arg_types);
756 OMPRTL__tgt_target_data_update_nowait,
757 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
758 OMPRTL__tgt_mapper_num_components,
759 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
760 // *base, void *begin, int64_t size, int64_t type);
761 OMPRTL__tgt_push_mapper_component,
762};
763
764/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
765/// region.
766class CleanupTy final : public EHScopeStack::Cleanup {
767 PrePostActionTy *Action;
768
769public:
770 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
771 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
772 if (!CGF.HaveInsertPoint())
773 return;
774 Action->Exit(CGF);
775 }
776};
777
778} // anonymous namespace
779
780void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
781 CodeGenFunction::RunCleanupsScope Scope(CGF);
782 if (PrePostAction
11.1
Field 'PrePostAction' is null
11.1
Field 'PrePostAction' is null
) {
12
Taking false branch
783 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
784 Callback(CodeGen, CGF, *PrePostAction);
785 } else {
786 PrePostActionTy Action;
787 Callback(CodeGen, CGF, Action);
13
Calling 'RegionCodeGenTy::CallbackFn'
788 }
789}
790
791/// Check if the combiner is a call to UDR combiner and if it is so return the
792/// UDR decl used for reduction.
793static const OMPDeclareReductionDecl *
794getReductionInit(const Expr *ReductionOp) {
795 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
796 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
797 if (const auto *DRE =
798 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
799 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
800 return DRD;
801 return nullptr;
802}
803
804static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
805 const OMPDeclareReductionDecl *DRD,
806 const Expr *InitOp,
807 Address Private, Address Original,
808 QualType Ty) {
809 if (DRD->getInitializer()) {
810 std::pair<llvm::Function *, llvm::Function *> Reduction =
811 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
812 const auto *CE = cast<CallExpr>(InitOp);
813 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
814 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
815 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
816 const auto *LHSDRE =
817 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
818 const auto *RHSDRE =
819 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
820 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
821 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
822 [=]() { return Private; });
823 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
824 [=]() { return Original; });
825 (void)PrivateScope.Privatize();
826 RValue Func = RValue::get(Reduction.second);
827 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
828 CGF.EmitIgnoredExpr(InitOp);
829 } else {
830 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
831 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
832 auto *GV = new llvm::GlobalVariable(
833 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
834 llvm::GlobalValue::PrivateLinkage, Init, Name);
835 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
836 RValue InitRVal;
837 switch (CGF.getEvaluationKind(Ty)) {
838 case TEK_Scalar:
839 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
840 break;
841 case TEK_Complex:
842 InitRVal =
843 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
844 break;
845 case TEK_Aggregate:
846 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
847 break;
848 }
849 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
850 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
851 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
852 /*IsInitializer=*/false);
853 }
854}
855
856/// Emit initialization of arrays of complex types.
857/// \param DestAddr Address of the array.
858/// \param Type Type of array.
859/// \param Init Initial expression of array.
860/// \param SrcAddr Address of the original array.
861static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
862 QualType Type, bool EmitDeclareReductionInit,
863 const Expr *Init,
864 const OMPDeclareReductionDecl *DRD,
865 Address SrcAddr = Address::invalid()) {
866 // Perform element-by-element initialization.
867 QualType ElementTy;
868
869 // Drill down to the base element type on both arrays.
870 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
871 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
872 DestAddr =
873 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
874 if (DRD)
875 SrcAddr =
876 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
877
878 llvm::Value *SrcBegin = nullptr;
879 if (DRD)
880 SrcBegin = SrcAddr.getPointer();
881 llvm::Value *DestBegin = DestAddr.getPointer();
882 // Cast from pointer to array type to pointer to single element.
883 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
884 // The basic structure here is a while-do loop.
885 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
886 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
887 llvm::Value *IsEmpty =
888 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
889 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
890
891 // Enter the loop body, making that address the current address.
892 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
893 CGF.EmitBlock(BodyBB);
894
895 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
896
897 llvm::PHINode *SrcElementPHI = nullptr;
898 Address SrcElementCurrent = Address::invalid();
899 if (DRD) {
900 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
901 "omp.arraycpy.srcElementPast");
902 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
903 SrcElementCurrent =
904 Address(SrcElementPHI,
905 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
906 }
907 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
908 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
909 DestElementPHI->addIncoming(DestBegin, EntryBB);
910 Address DestElementCurrent =
911 Address(DestElementPHI,
912 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
913
914 // Emit copy.
915 {
916 CodeGenFunction::RunCleanupsScope InitScope(CGF);
917 if (EmitDeclareReductionInit) {
918 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
919 SrcElementCurrent, ElementTy);
920 } else
921 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
922 /*IsInitializer=*/false);
923 }
924
925 if (DRD) {
926 // Shift the address forward by one element.
927 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
928 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
929 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
930 }
931
932 // Shift the address forward by one element.
933 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
934 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
935 // Check whether we've reached the end.
936 llvm::Value *Done =
937 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
938 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
939 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
940
941 // Done.
942 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
943}
944
945LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
946 return CGF.EmitOMPSharedLValue(E);
947}
948
949LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
950 const Expr *E) {
951 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
952 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
953 return LValue();
954}
955
956void ReductionCodeGen::emitAggregateInitialization(
957 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
958 const OMPDeclareReductionDecl *DRD) {
959 // Emit VarDecl with copy init for arrays.
960 // Get the address of the original variable captured in current
961 // captured region.
962 const auto *PrivateVD =
963 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
964 bool EmitDeclareReductionInit =
965 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
966 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
967 EmitDeclareReductionInit,
968 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
969 : PrivateVD->getInit(),
970 DRD, SharedLVal.getAddress(CGF));
971}
972
973ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
974 ArrayRef<const Expr *> Privates,
975 ArrayRef<const Expr *> ReductionOps) {
976 ClausesData.reserve(Shareds.size());
977 SharedAddresses.reserve(Shareds.size());
978 Sizes.reserve(Shareds.size());
979 BaseDecls.reserve(Shareds.size());
980 auto IPriv = Privates.begin();
981 auto IRed = ReductionOps.begin();
982 for (const Expr *Ref : Shareds) {
983 ClausesData.emplace_back(Ref, *IPriv, *IRed);
984 std::advance(IPriv, 1);
985 std::advance(IRed, 1);
986 }
987}
988
989void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
990 assert(SharedAddresses.size() == N &&((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 991, __PRETTY_FUNCTION__))
991 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 991, __PRETTY_FUNCTION__))
;
992 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
993 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
994 SharedAddresses.emplace_back(First, Second);
995}
996
997void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
998 const auto *PrivateVD =
999 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1000 QualType PrivateType = PrivateVD->getType();
1001 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1002 if (!PrivateType->isVariablyModifiedType()) {
1003 Sizes.emplace_back(
1004 CGF.getTypeSize(
1005 SharedAddresses[N].first.getType().getNonReferenceType()),
1006 nullptr);
1007 return;
1008 }
1009 llvm::Value *Size;
1010 llvm::Value *SizeInChars;
1011 auto *ElemType = cast<llvm::PointerType>(
1012 SharedAddresses[N].first.getPointer(CGF)->getType())
1013 ->getElementType();
1014 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1015 if (AsArraySection) {
1016 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1017 SharedAddresses[N].first.getPointer(CGF));
1018 Size = CGF.Builder.CreateNUWAdd(
1019 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1020 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1021 } else {
1022 SizeInChars = CGF.getTypeSize(
1023 SharedAddresses[N].first.getType().getNonReferenceType());
1024 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1025 }
1026 Sizes.emplace_back(SizeInChars, Size);
1027 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1028 CGF,
1029 cast<OpaqueValueExpr>(
1030 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1031 RValue::get(Size));
1032 CGF.EmitVariablyModifiedType(PrivateType);
1033}
1034
1035void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1036 llvm::Value *Size) {
1037 const auto *PrivateVD =
1038 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1039 QualType PrivateType = PrivateVD->getType();
1040 if (!PrivateType->isVariablyModifiedType()) {
1041 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1043, __PRETTY_FUNCTION__))
1042 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1043, __PRETTY_FUNCTION__))
1043 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1043, __PRETTY_FUNCTION__))
;
1044 return;
1045 }
1046 CodeGenFunction::OpaqueValueMapping OpaqueMap(
1047 CGF,
1048 cast<OpaqueValueExpr>(
1049 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1050 RValue::get(Size));
1051 CGF.EmitVariablyModifiedType(PrivateType);
1052}
1053
1054void ReductionCodeGen::emitInitialization(
1055 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1056 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1057 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1057, __PRETTY_FUNCTION__))
;
1058 const auto *PrivateVD =
1059 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1060 const OMPDeclareReductionDecl *DRD =
1061 getReductionInit(ClausesData[N].ReductionOp);
1062 QualType PrivateType = PrivateVD->getType();
1063 PrivateAddr = CGF.Builder.CreateElementBitCast(
1064 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1065 QualType SharedType = SharedAddresses[N].first.getType();
1066 SharedLVal = CGF.MakeAddrLValue(
1067 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
1068 CGF.ConvertTypeForMem(SharedType)),
1069 SharedType, SharedAddresses[N].first.getBaseInfo(),
1070 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1071 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1072 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1073 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1074 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1075 PrivateAddr, SharedLVal.getAddress(CGF),
1076 SharedLVal.getType());
1077 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1078 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1079 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1080 PrivateVD->getType().getQualifiers(),
1081 /*IsInitializer=*/false);
1082 }
1083}
1084
1085bool ReductionCodeGen::needCleanups(unsigned N) {
1086 const auto *PrivateVD =
1087 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1088 QualType PrivateType = PrivateVD->getType();
1089 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1090 return DTorKind != QualType::DK_none;
1091}
1092
1093void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1094 Address PrivateAddr) {
1095 const auto *PrivateVD =
1096 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1097 QualType PrivateType = PrivateVD->getType();
1098 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1099 if (needCleanups(N)) {
1100 PrivateAddr = CGF.Builder.CreateElementBitCast(
1101 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1102 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1103 }
1104}
1105
1106static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1107 LValue BaseLV) {
1108 BaseTy = BaseTy.getNonReferenceType();
1109 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1110 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1111 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1112 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
1113 } else {
1114 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
1115 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1116 }
1117 BaseTy = BaseTy->getPointeeType();
1118 }
1119 return CGF.MakeAddrLValue(
1120 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
1121 CGF.ConvertTypeForMem(ElTy)),
1122 BaseLV.getType(), BaseLV.getBaseInfo(),
1123 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1124}
1125
1126static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1127 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1128 llvm::Value *Addr) {
1129 Address Tmp = Address::invalid();
1130 Address TopTmp = Address::invalid();
1131 Address MostTopTmp = Address::invalid();
1132 BaseTy = BaseTy.getNonReferenceType();
1133 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1134 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1135 Tmp = CGF.CreateMemTemp(BaseTy);
1136 if (TopTmp.isValid())
1137 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1138 else
1139 MostTopTmp = Tmp;
1140 TopTmp = Tmp;
1141 BaseTy = BaseTy->getPointeeType();
1142 }
1143 llvm::Type *Ty = BaseLVType;
1144 if (Tmp.isValid())
1145 Ty = Tmp.getElementType();
1146 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1147 if (Tmp.isValid()) {
1148 CGF.Builder.CreateStore(Addr, Tmp);
1149 return MostTopTmp;
1150 }
1151 return Address(Addr, BaseLVAlignment);
1152}
1153
1154static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1155 const VarDecl *OrigVD = nullptr;
1156 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1157 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1158 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1159 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1160 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1161 Base = TempASE->getBase()->IgnoreParenImpCasts();
1162 DE = cast<DeclRefExpr>(Base);
1163 OrigVD = cast<VarDecl>(DE->getDecl());
1164 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1165 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1166 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1167 Base = TempASE->getBase()->IgnoreParenImpCasts();
1168 DE = cast<DeclRefExpr>(Base);
1169 OrigVD = cast<VarDecl>(DE->getDecl());
1170 }
1171 return OrigVD;
1172}
1173
1174Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1175 Address PrivateAddr) {
1176 const DeclRefExpr *DE;
1177 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1178 BaseDecls.emplace_back(OrigVD);
1179 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1180 LValue BaseLValue =
1181 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1182 OriginalBaseLValue);
1183 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1184 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1185 llvm::Value *PrivatePointer =
1186 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1187 PrivateAddr.getPointer(),
1188 SharedAddresses[N].first.getAddress(CGF).getType());
1189 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1190 return castToBase(CGF, OrigVD->getType(),
1191 SharedAddresses[N].first.getType(),
1192 OriginalBaseLValue.getAddress(CGF).getType(),
1193 OriginalBaseLValue.getAlignment(), Ptr);
1194 }
1195 BaseDecls.emplace_back(
1196 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1197 return PrivateAddr;
1198}
1199
1200bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1201 const OMPDeclareReductionDecl *DRD =
1202 getReductionInit(ClausesData[N].ReductionOp);
1203 return DRD && DRD->getInitializer();
1204}
1205
1206LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1207 return CGF.EmitLoadOfPointerLValue(
1208 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1209 getThreadIDVariable()->getType()->castAs<PointerType>());
1210}
1211
1212void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1213 if (!CGF.HaveInsertPoint())
1214 return;
1215 // 1.2.2 OpenMP Language Terminology
1216 // Structured block - An executable statement with a single entry at the
1217 // top and a single exit at the bottom.
1218 // The point of exit cannot be a branch out of the structured block.
1219 // longjmp() and throw() must not violate the entry/exit criteria.
1220 CGF.EHStack.pushTerminate();
1221 CodeGen(CGF);
1222 CGF.EHStack.popTerminate();
1223}
1224
1225LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1226 CodeGenFunction &CGF) {
1227 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1228 getThreadIDVariable()->getType(),
1229 AlignmentSource::Decl);
1230}
1231
1232static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1233 QualType FieldTy) {
1234 auto *Field = FieldDecl::Create(
1235 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1236 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1237 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1238 Field->setAccess(AS_public);
1239 DC->addDecl(Field);
1240 return Field;
1241}
1242
1243CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1244 StringRef Separator)
1245 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1246 OffloadEntriesInfoManager(CGM) {
1247 ASTContext &C = CGM.getContext();
1248 RecordDecl *RD = C.buildImplicitRecord("ident_t");
1249 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1250 RD->startDefinition();
1251 // reserved_1
1252 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1253 // flags
1254 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1255 // reserved_2
1256 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1257 // reserved_3
1258 addFieldToRecordDecl(C, RD, KmpInt32Ty);
1259 // psource
1260 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1261 RD->completeDefinition();
1262 IdentQTy = C.getRecordType(RD);
1263 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1264 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1265
1266 loadOffloadInfoMetadata();
1267}
1268
1269bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1270 const GlobalDecl &OldGD,
1271 llvm::GlobalValue *OrigAddr,
1272 bool IsForDefinition) {
1273 // Emit at least a definition for the aliasee if the the address of the
1274 // original function is requested.
1275 if (IsForDefinition || OrigAddr)
1276 (void)CGM.GetAddrOfGlobal(NewGD);
1277 StringRef NewMangledName = CGM.getMangledName(NewGD);
1278 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1279 if (Addr && !Addr->isDeclaration()) {
1280 const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1281 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(NewGD);
1282 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1283
1284 // Create a reference to the named value. This ensures that it is emitted
1285 // if a deferred decl.
1286 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1287
1288 // Create the new alias itself, but don't set a name yet.
1289 auto *GA =
1290 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1291
1292 if (OrigAddr) {
1293 assert(OrigAddr->isDeclaration() && "Expected declaration")((OrigAddr->isDeclaration() && "Expected declaration"
) ? static_cast<void> (0) : __assert_fail ("OrigAddr->isDeclaration() && \"Expected declaration\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1293, __PRETTY_FUNCTION__))
;
1294
1295 GA->takeName(OrigAddr);
1296 OrigAddr->replaceAllUsesWith(
1297 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1298 OrigAddr->eraseFromParent();
1299 } else {
1300 GA->setName(CGM.getMangledName(OldGD));
1301 }
1302
1303 // Set attributes which are particular to an alias; this is a
1304 // specialization of the attributes which may be set on a global function.
1305 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1306 D->isWeakImported())
1307 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1308
1309 CGM.SetCommonAttributes(OldGD, GA);
1310 return true;
1311 }
1312 return false;
1313}
1314
1315void CGOpenMPRuntime::clear() {
1316 InternalVars.clear();
1317 // Clean non-target variable declarations possibly used only in debug info.
1318 for (const auto &Data : EmittedNonTargetVariables) {
1319 if (!Data.getValue().pointsToAliveValue())
1320 continue;
1321 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1322 if (!GV)
1323 continue;
1324 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1325 continue;
1326 GV->eraseFromParent();
1327 }
1328 // Emit aliases for the deferred aliasees.
1329 for (const auto &Pair : DeferredVariantFunction) {
1330 StringRef MangledName = CGM.getMangledName(Pair.second.second);
1331 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1332 // If not able to emit alias, just emit original declaration.
1333 (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1334 /*IsForDefinition=*/false);
1335 }
1336}
1337
1338std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1339 SmallString<128> Buffer;
1340 llvm::raw_svector_ostream OS(Buffer);
1341 StringRef Sep = FirstSeparator;
1342 for (StringRef Part : Parts) {
1343 OS << Sep << Part;
1344 Sep = Separator;
1345 }
1346 return OS.str();
1347}
1348
1349static llvm::Function *
1350emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1351 const Expr *CombinerInitializer, const VarDecl *In,
1352 const VarDecl *Out, bool IsCombiner) {
1353 // void .omp_combiner.(Ty *in, Ty *out);
1354 ASTContext &C = CGM.getContext();
1355 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1356 FunctionArgList Args;
1357 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1358 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1359 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1360 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1361 Args.push_back(&OmpOutParm);
1362 Args.push_back(&OmpInParm);
1363 const CGFunctionInfo &FnInfo =
1364 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1365 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1366 std::string Name = CGM.getOpenMPRuntime().getName(
1367 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1368 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1369 Name, &CGM.getModule());
1370 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1371 if (CGM.getLangOpts().Optimize) {
1372 Fn->removeFnAttr(llvm::Attribute::NoInline);
1373 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1374 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1375 }
1376 CodeGenFunction CGF(CGM);
1377 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1378 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1379 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1380 Out->getLocation());
1381 CodeGenFunction::OMPPrivateScope Scope(CGF);
1382 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1383 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1384 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1385 .getAddress(CGF);
1386 });
1387 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1388 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1389 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1390 .getAddress(CGF);
1391 });
1392 (void)Scope.Privatize();
1393 if (!IsCombiner && Out->hasInit() &&
1394 !CGF.isTrivialInitializer(Out->getInit())) {
1395 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1396 Out->getType().getQualifiers(),
1397 /*IsInitializer=*/true);
1398 }
1399 if (CombinerInitializer)
1400 CGF.EmitIgnoredExpr(CombinerInitializer);
1401 Scope.ForceCleanup();
1402 CGF.FinishFunction();
1403 return Fn;
1404}
1405
1406void CGOpenMPRuntime::emitUserDefinedReduction(
1407 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1408 if (UDRMap.count(D) > 0)
1409 return;
1410 llvm::Function *Combiner = emitCombinerOrInitializer(
1411 CGM, D->getType(), D->getCombiner(),
1412 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1413 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1414 /*IsCombiner=*/true);
1415 llvm::Function *Initializer = nullptr;
1416 if (const Expr *Init = D->getInitializer()) {
1417 Initializer = emitCombinerOrInitializer(
1418 CGM, D->getType(),
1419 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1420 : nullptr,
1421 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1422 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1423 /*IsCombiner=*/false);
1424 }
1425 UDRMap.try_emplace(D, Combiner, Initializer);
1426 if (CGF) {
1427 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1428 Decls.second.push_back(D);
1429 }
1430}
1431
1432std::pair<llvm::Function *, llvm::Function *>
1433CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1434 auto I = UDRMap.find(D);
1435 if (I != UDRMap.end())
1436 return I->second;
1437 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1438 return UDRMap.lookup(D);
1439}
1440
1441// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1442// Builder if one is present.
1443struct PushAndPopStackRAII {
1444 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1445 bool HasCancel)
1446 : OMPBuilder(OMPBuilder) {
1447 if (!OMPBuilder)
1448 return;
1449
1450 // The following callback is the crucial part of clangs cleanup process.
1451 //
1452 // NOTE:
1453 // Once the OpenMPIRBuilder is used to create parallel regions (and
1454 // similar), the cancellation destination (Dest below) is determined via
1455 // IP. That means if we have variables to finalize we split the block at IP,
1456 // use the new block (=BB) as destination to build a JumpDest (via
1457 // getJumpDestInCurrentScope(BB)) which then is fed to
1458 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1459 // to push & pop an FinalizationInfo object.
1460 // The FiniCB will still be needed but at the point where the
1461 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1462 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1463 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1464, __PRETTY_FUNCTION__))
1464 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1464, __PRETTY_FUNCTION__))
;
1465 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1466 CGF.Builder.restoreIP(IP);
1467 CodeGenFunction::JumpDest Dest =
1468 CGF.getOMPCancelDestination(OMPD_parallel);
1469 CGF.EmitBranchThroughCleanup(Dest);
1470 };
1471
1472 // TODO: Remove this once we emit parallel regions through the
1473 // OpenMPIRBuilder as it can do this setup internally.
1474 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1475 {FiniCB, OMPD_parallel, HasCancel});
1476 OMPBuilder->pushFinalizationCB(std::move(FI));
1477 }
1478 ~PushAndPopStackRAII() {
1479 if (OMPBuilder)
1480 OMPBuilder->popFinalizationCB();
1481 }
1482 llvm::OpenMPIRBuilder *OMPBuilder;
1483};
1484
1485static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1486 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1487 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1488 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1489 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1490, __PRETTY_FUNCTION__))
1490 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1490, __PRETTY_FUNCTION__))
;
1491 CodeGenFunction CGF(CGM, true);
1492 bool HasCancel = false;
1493 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1494 HasCancel = OPD->hasCancel();
1495 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1496 HasCancel = OPSD->hasCancel();
1497 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1498 HasCancel = OPFD->hasCancel();
1499 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1500 HasCancel = OPFD->hasCancel();
1501 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1502 HasCancel = OPFD->hasCancel();
1503 else if (const auto *OPFD =
1504 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1505 HasCancel = OPFD->hasCancel();
1506 else if (const auto *OPFD =
1507 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1508 HasCancel = OPFD->hasCancel();
1509
1510 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1511 // parallel region to make cancellation barriers work properly.
1512 llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder();
1513 PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
1514 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1515 HasCancel, OutlinedHelperName);
1516 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1517 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1518}
1519
1520llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1521 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1522 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1523 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1524 return emitParallelOrTeamsOutlinedFunction(
1525 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1526}
1527
1528llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1529 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1530 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1531 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1532 return emitParallelOrTeamsOutlinedFunction(
1533 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1534}
1535
1536llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1537 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1538 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1539 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1540 bool Tied, unsigned &NumberOfParts) {
1541 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1542 PrePostActionTy &) {
1543 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1544 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1545 llvm::Value *TaskArgs[] = {
1546 UpLoc, ThreadID,
1547 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1548 TaskTVar->getType()->castAs<PointerType>())
1549 .getPointer(CGF)};
1550 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1551 };
1552 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1553 UntiedCodeGen);
1554 CodeGen.setAction(Action);
1555 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1556, __PRETTY_FUNCTION__))
1556 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1556, __PRETTY_FUNCTION__))
;
1557 const OpenMPDirectiveKind Region =
1558 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1559 : OMPD_task;
1560 const CapturedStmt *CS = D.getCapturedStmt(Region);
1561 const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1562 CodeGenFunction CGF(CGM, true);
1563 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1564 InnermostKind,
1565 TD ? TD->hasCancel() : false, Action);
1566 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1567 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1568 if (!Tied)
1569 NumberOfParts = Action.getNumberOfParts();
1570 return Res;
1571}
1572
1573static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1574 const RecordDecl *RD, const CGRecordLayout &RL,
1575 ArrayRef<llvm::Constant *> Data) {
1576 llvm::StructType *StructTy = RL.getLLVMType();
1577 unsigned PrevIdx = 0;
1578 ConstantInitBuilder CIBuilder(CGM);
1579 auto DI = Data.begin();
1580 for (const FieldDecl *FD : RD->fields()) {
1581 unsigned Idx = RL.getLLVMFieldNo(FD);
1582 // Fill the alignment.
1583 for (unsigned I = PrevIdx; I < Idx; ++I)
1584 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1585 PrevIdx = Idx + 1;
1586 Fields.add(*DI);
1587 ++DI;
1588 }
1589}
1590
1591template <class... As>
1592static llvm::GlobalVariable *
1593createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1594 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1595 As &&... Args) {
1596 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1597 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1598 ConstantInitBuilder CIBuilder(CGM);
1599 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1600 buildStructValue(Fields, CGM, RD, RL, Data);
1601 return Fields.finishAndCreateGlobal(
1602 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1603 std::forward<As>(Args)...);
1604}
1605
1606template <typename T>
1607static void
1608createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1609 ArrayRef<llvm::Constant *> Data,
1610 T &Parent) {
1611 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1612 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1613 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1614 buildStructValue(Fields, CGM, RD, RL, Data);
1615 Fields.finishAndAddTo(Parent);
1616}
1617
1618Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1619 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1620 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1621 FlagsTy FlagsKey(Flags, Reserved2Flags);
1622 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1623 if (!Entry) {
1624 if (!DefaultOpenMPPSource) {
1625 // Initialize default location for psource field of ident_t structure of
1626 // all ident_t objects. Format is ";file;function;line;column;;".
1627 // Taken from
1628 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1629 DefaultOpenMPPSource =
1630 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1631 DefaultOpenMPPSource =
1632 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1633 }
1634
1635 llvm::Constant *Data[] = {
1636 llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1637 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1638 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1639 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1640 llvm::GlobalValue *DefaultOpenMPLocation =
1641 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1642 llvm::GlobalValue::PrivateLinkage);
1643 DefaultOpenMPLocation->setUnnamedAddr(
1644 llvm::GlobalValue::UnnamedAddr::Global);
1645
1646 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1647 }
1648 return Address(Entry, Align);
1649}
1650
1651void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1652 bool AtCurrentPoint) {
1653 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1654 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1654, __PRETTY_FUNCTION__))
;
1655
1656 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1657 if (AtCurrentPoint) {
1658 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1659 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1660 } else {
1661 Elem.second.ServiceInsertPt =
1662 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1663 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1664 }
1665}
1666
1667void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1668 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1669 if (Elem.second.ServiceInsertPt) {
1670 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1671 Elem.second.ServiceInsertPt = nullptr;
1672 Ptr->eraseFromParent();
1673 }
1674}
1675
1676llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1677 SourceLocation Loc,
1678 unsigned Flags) {
1679 Flags |= OMP_IDENT_KMPC;
1680 // If no debug info is generated - return global default location.
1681 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1682 Loc.isInvalid())
1683 return getOrCreateDefaultLocation(Flags).getPointer();
1684
1685 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1685, __PRETTY_FUNCTION__))
;
1686
1687 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1688 Address LocValue = Address::invalid();
1689 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1690 if (I != OpenMPLocThreadIDMap.end())
1691 LocValue = Address(I->second.DebugLoc, Align);
1692
1693 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1694 // GetOpenMPThreadID was called before this routine.
1695 if (!LocValue.isValid()) {
1696 // Generate "ident_t .kmpc_loc.addr;"
1697 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1698 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1699 Elem.second.DebugLoc = AI.getPointer();
1700 LocValue = AI;
1701
1702 if (!Elem.second.ServiceInsertPt)
1703 setLocThreadIdInsertPt(CGF);
1704 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1705 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1706 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1707 CGF.getTypeSize(IdentQTy));
1708 }
1709
1710 // char **psource = &.kmpc_loc_<flags>.addr.psource;
1711 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1712 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1713 LValue PSource =
1714 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1715
1716 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1717 if (OMPDebugLoc == nullptr) {
1718 SmallString<128> Buffer2;
1719 llvm::raw_svector_ostream OS2(Buffer2);
1720 // Build debug location
1721 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1722 OS2 << ";" << PLoc.getFilename() << ";";
1723 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1724 OS2 << FD->getQualifiedNameAsString();
1725 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1726 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1727 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1728 }
1729 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1730 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1731
1732 // Our callers always pass this to a runtime function, so for
1733 // convenience, go ahead and return a naked pointer.
1734 return LocValue.getPointer();
1735}
1736
1737llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1738 SourceLocation Loc) {
1739 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1739, __PRETTY_FUNCTION__))
;
1740
1741 llvm::Value *ThreadID = nullptr;
1742 // Check whether we've already cached a load of the thread id in this
1743 // function.
1744 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1745 if (I != OpenMPLocThreadIDMap.end()) {
1746 ThreadID = I->second.ThreadID;
1747 if (ThreadID != nullptr)
1748 return ThreadID;
1749 }
1750 // If exceptions are enabled, do not use parameter to avoid possible crash.
1751 if (auto *OMPRegionInfo =
1752 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1753 if (OMPRegionInfo->getThreadIDVariable()) {
1754 // Check if this an outlined function with thread id passed as argument.
1755 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1756 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1757 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1758 !CGF.getLangOpts().CXXExceptions ||
1759 CGF.Builder.GetInsertBlock() == TopBlock ||
1760 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1761 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1762 TopBlock ||
1763 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1764 CGF.Builder.GetInsertBlock()) {
1765 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1766 // If value loaded in entry block, cache it and use it everywhere in
1767 // function.
1768 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1769 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1770 Elem.second.ThreadID = ThreadID;
1771 }
1772 return ThreadID;
1773 }
1774 }
1775 }
1776
1777 // This is not an outlined function region - need to call __kmpc_int32
1778 // kmpc_global_thread_num(ident_t *loc).
1779 // Generate thread id value and cache this value for use across the
1780 // function.
1781 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1782 if (!Elem.second.ServiceInsertPt)
1783 setLocThreadIdInsertPt(CGF);
1784 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1785 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1786 llvm::CallInst *Call = CGF.Builder.CreateCall(
1787 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1788 emitUpdateLocation(CGF, Loc));
1789 Call->setCallingConv(CGF.getRuntimeCC());
1790 Elem.second.ThreadID = Call;
1791 return Call;
1792}
1793
1794void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1795 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1795, __PRETTY_FUNCTION__))
;
1796 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1797 clearLocThreadIdInsertPt(CGF);
1798 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1799 }
1800 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1801 for(auto *D : FunctionUDRMap[CGF.CurFn])
1802 UDRMap.erase(D);
1803 FunctionUDRMap.erase(CGF.CurFn);
1804 }
1805 auto I = FunctionUDMMap.find(CGF.CurFn);
1806 if (I != FunctionUDMMap.end()) {
1807 for(auto *D : I->second)
1808 UDMMap.erase(D);
1809 FunctionUDMMap.erase(I);
1810 }
1811}
1812
1813llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1814 return IdentTy->getPointerTo();
1815}
1816
1817llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1818 if (!Kmpc_MicroTy) {
1819 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1820 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1821 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1822 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1823 }
1824 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1825}
1826
1827llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1828 llvm::FunctionCallee RTLFn = nullptr;
1829 switch (static_cast<OpenMPRTLFunction>(Function)) {
1830 case OMPRTL__kmpc_fork_call: {
1831 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1832 // microtask, ...);
1833 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1834 getKmpc_MicroPointerTy()};
1835 auto *FnTy =
1836 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1837 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1838 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1839 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1840 llvm::LLVMContext &Ctx = F->getContext();
1841 llvm::MDBuilder MDB(Ctx);
1842 // Annotate the callback behavior of the __kmpc_fork_call:
1843 // - The callback callee is argument number 2 (microtask).
1844 // - The first two arguments of the callback callee are unknown (-1).
1845 // - All variadic arguments to the __kmpc_fork_call are passed to the
1846 // callback callee.
1847 F->addMetadata(
1848 llvm::LLVMContext::MD_callback,
1849 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1850 2, {-1, -1},
1851 /* VarArgsArePassed */ true)}));
1852 }
1853 }
1854 break;
1855 }
1856 case OMPRTL__kmpc_global_thread_num: {
1857 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1858 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1859 auto *FnTy =
1860 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1861 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1862 break;
1863 }
1864 case OMPRTL__kmpc_threadprivate_cached: {
1865 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1866 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1867 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1868 CGM.VoidPtrTy, CGM.SizeTy,
1869 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1870 auto *FnTy =
1871 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1872 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1873 break;
1874 }
1875 case OMPRTL__kmpc_critical: {
1876 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1877 // kmp_critical_name *crit);
1878 llvm::Type *TypeParams[] = {
1879 getIdentTyPointerTy(), CGM.Int32Ty,
1880 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1881 auto *FnTy =
1882 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1883 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1884 break;
1885 }
1886 case OMPRTL__kmpc_critical_with_hint: {
1887 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1888 // kmp_critical_name *crit, uintptr_t hint);
1889 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1890 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1891 CGM.IntPtrTy};
1892 auto *FnTy =
1893 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1894 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1895 break;
1896 }
1897 case OMPRTL__kmpc_threadprivate_register: {
1898 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1899 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1900 // typedef void *(*kmpc_ctor)(void *);
1901 auto *KmpcCtorTy =
1902 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1903 /*isVarArg*/ false)->getPointerTo();
1904 // typedef void *(*kmpc_cctor)(void *, void *);
1905 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1906 auto *KmpcCopyCtorTy =
1907 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1908 /*isVarArg*/ false)
1909 ->getPointerTo();
1910 // typedef void (*kmpc_dtor)(void *);
1911 auto *KmpcDtorTy =
1912 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1913 ->getPointerTo();
1914 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1915 KmpcCopyCtorTy, KmpcDtorTy};
1916 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1917 /*isVarArg*/ false);
1918 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1919 break;
1920 }
1921 case OMPRTL__kmpc_end_critical: {
1922 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1923 // kmp_critical_name *crit);
1924 llvm::Type *TypeParams[] = {
1925 getIdentTyPointerTy(), CGM.Int32Ty,
1926 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1927 auto *FnTy =
1928 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1929 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1930 break;
1931 }
1932 case OMPRTL__kmpc_cancel_barrier: {
1933 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1934 // global_tid);
1935 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1936 auto *FnTy =
1937 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1938 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1939 break;
1940 }
1941 case OMPRTL__kmpc_barrier: {
1942 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1943 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1944 auto *FnTy =
1945 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1946 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1947 break;
1948 }
1949 case OMPRTL__kmpc_for_static_fini: {
1950 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1951 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1952 auto *FnTy =
1953 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1954 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1955 break;
1956 }
1957 case OMPRTL__kmpc_push_num_threads: {
1958 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1959 // kmp_int32 num_threads)
1960 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1961 CGM.Int32Ty};
1962 auto *FnTy =
1963 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1964 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1965 break;
1966 }
1967 case OMPRTL__kmpc_serialized_parallel: {
1968 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1969 // global_tid);
1970 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1971 auto *FnTy =
1972 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1973 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1974 break;
1975 }
1976 case OMPRTL__kmpc_end_serialized_parallel: {
1977 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1978 // global_tid);
1979 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1980 auto *FnTy =
1981 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1982 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1983 break;
1984 }
1985 case OMPRTL__kmpc_flush: {
1986 // Build void __kmpc_flush(ident_t *loc);
1987 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1988 auto *FnTy =
1989 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1990 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1991 break;
1992 }
1993 case OMPRTL__kmpc_master: {
1994 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1995 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1996 auto *FnTy =
1997 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1998 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1999 break;
2000 }
2001 case OMPRTL__kmpc_end_master: {
2002 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
2003 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2004 auto *FnTy =
2005 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2006 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
2007 break;
2008 }
2009 case OMPRTL__kmpc_omp_taskyield: {
2010 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
2011 // int end_part);
2012 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2013 auto *FnTy =
2014 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2015 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
2016 break;
2017 }
2018 case OMPRTL__kmpc_single: {
2019 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
2020 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2021 auto *FnTy =
2022 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2023 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
2024 break;
2025 }
2026 case OMPRTL__kmpc_end_single: {
2027 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
2028 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2029 auto *FnTy =
2030 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2031 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
2032 break;
2033 }
2034 case OMPRTL__kmpc_omp_task_alloc: {
2035 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
2036 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2037 // kmp_routine_entry_t *task_entry);
2038 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2039, __PRETTY_FUNCTION__))
2039 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2039, __PRETTY_FUNCTION__))
;
2040 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2041 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
2042 // Return void * and then cast to particular kmp_task_t type.
2043 auto *FnTy =
2044 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2045 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
2046 break;
2047 }
2048 case OMPRTL__kmpc_omp_target_task_alloc: {
2049 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
2050 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2051 // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
2052 assert(KmpRoutineEntryPtrTy != nullptr &&((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2053, __PRETTY_FUNCTION__))
2053 "Type kmp_routine_entry_t must be created.")((KmpRoutineEntryPtrTy != nullptr && "Type kmp_routine_entry_t must be created."
) ? static_cast<void> (0) : __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2053, __PRETTY_FUNCTION__))
;
2054 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2055 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
2056 CGM.Int64Ty};
2057 // Return void * and then cast to particular kmp_task_t type.
2058 auto *FnTy =
2059 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2060 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2061 break;
2062 }
2063 case OMPRTL__kmpc_omp_task: {
2064 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2065 // *new_task);
2066 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2067 CGM.VoidPtrTy};
2068 auto *FnTy =
2069 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2070 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2071 break;
2072 }
2073 case OMPRTL__kmpc_copyprivate: {
2074 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2075 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2076 // kmp_int32 didit);
2077 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2078 auto *CpyFnTy =
2079 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2080 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2081 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2082 CGM.Int32Ty};
2083 auto *FnTy =
2084 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2085 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2086 break;
2087 }
2088 case OMPRTL__kmpc_reduce: {
2089 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2090 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2091 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2092 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2093 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2094 /*isVarArg=*/false);
2095 llvm::Type *TypeParams[] = {
2096 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2097 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2098 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2099 auto *FnTy =
2100 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2101 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2102 break;
2103 }
2104 case OMPRTL__kmpc_reduce_nowait: {
2105 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2106 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2107 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2108 // *lck);
2109 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2110 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2111 /*isVarArg=*/false);
2112 llvm::Type *TypeParams[] = {
2113 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2114 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2115 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2116 auto *FnTy =
2117 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2118 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2119 break;
2120 }
2121 case OMPRTL__kmpc_end_reduce: {
2122 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2123 // kmp_critical_name *lck);
2124 llvm::Type *TypeParams[] = {
2125 getIdentTyPointerTy(), CGM.Int32Ty,
2126 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2127 auto *FnTy =
2128 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2129 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2130 break;
2131 }
2132 case OMPRTL__kmpc_end_reduce_nowait: {
2133 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2134 // kmp_critical_name *lck);
2135 llvm::Type *TypeParams[] = {
2136 getIdentTyPointerTy(), CGM.Int32Ty,
2137 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2138 auto *FnTy =
2139 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2140 RTLFn =
2141 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2142 break;
2143 }
2144 case OMPRTL__kmpc_omp_task_begin_if0: {
2145 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2146 // *new_task);
2147 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2148 CGM.VoidPtrTy};
2149 auto *FnTy =
2150 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2151 RTLFn =
2152 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2153 break;
2154 }
2155 case OMPRTL__kmpc_omp_task_complete_if0: {
2156 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2157 // *new_task);
2158 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2159 CGM.VoidPtrTy};
2160 auto *FnTy =
2161 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2162 RTLFn = CGM.CreateRuntimeFunction(FnTy,
2163 /*Name=*/"__kmpc_omp_task_complete_if0");
2164 break;
2165 }
2166 case OMPRTL__kmpc_ordered: {
2167 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2168 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2169 auto *FnTy =
2170 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2171 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2172 break;
2173 }
2174 case OMPRTL__kmpc_end_ordered: {
2175 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2176 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2177 auto *FnTy =
2178 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2179 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2180 break;
2181 }
2182 case OMPRTL__kmpc_omp_taskwait: {
2183 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2184 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2185 auto *FnTy =
2186 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2187 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2188 break;
2189 }
2190 case OMPRTL__kmpc_taskgroup: {
2191 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2192 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2193 auto *FnTy =
2194 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2195 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2196 break;
2197 }
2198 case OMPRTL__kmpc_end_taskgroup: {
2199 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2200 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2201 auto *FnTy =
2202 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2203 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2204 break;
2205 }
2206 case OMPRTL__kmpc_push_proc_bind: {
2207 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2208 // int proc_bind)
2209 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2210 auto *FnTy =
2211 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2212 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2213 break;
2214 }
2215 case OMPRTL__kmpc_omp_task_with_deps: {
2216 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2217 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2218 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2219 llvm::Type *TypeParams[] = {
2220 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2221 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
2222 auto *FnTy =
2223 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2224 RTLFn =
2225 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2226 break;
2227 }
2228 case OMPRTL__kmpc_omp_wait_deps: {
2229 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2230 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2231 // kmp_depend_info_t *noalias_dep_list);
2232 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2233 CGM.Int32Ty, CGM.VoidPtrTy,
2234 CGM.Int32Ty, CGM.VoidPtrTy};
2235 auto *FnTy =
2236 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2237 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2238 break;
2239 }
2240 case OMPRTL__kmpc_cancellationpoint: {
2241 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2242 // global_tid, kmp_int32 cncl_kind)
2243 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2244 auto *FnTy =
2245 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2246 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2247 break;
2248 }
2249 case OMPRTL__kmpc_cancel: {
2250 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2251 // kmp_int32 cncl_kind)
2252 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2253 auto *FnTy =
2254 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2255 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2256 break;
2257 }
2258 case OMPRTL__kmpc_push_num_teams: {
2259 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2260 // kmp_int32 num_teams, kmp_int32 num_threads)
2261 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2262 CGM.Int32Ty};
2263 auto *FnTy =
2264 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2265 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2266 break;
2267 }
2268 case OMPRTL__kmpc_fork_teams: {
2269 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2270 // microtask, ...);
2271 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2272 getKmpc_MicroPointerTy()};
2273 auto *FnTy =
2274 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2275 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2276 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2277 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2278 llvm::LLVMContext &Ctx = F->getContext();
2279 llvm::MDBuilder MDB(Ctx);
2280 // Annotate the callback behavior of the __kmpc_fork_teams:
2281 // - The callback callee is argument number 2 (microtask).
2282 // - The first two arguments of the callback callee are unknown (-1).
2283 // - All variadic arguments to the __kmpc_fork_teams are passed to the
2284 // callback callee.
2285 F->addMetadata(
2286 llvm::LLVMContext::MD_callback,
2287 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2288 2, {-1, -1},
2289 /* VarArgsArePassed */ true)}));
2290 }
2291 }
2292 break;
2293 }
2294 case OMPRTL__kmpc_taskloop: {
2295 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2296 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2297 // sched, kmp_uint64 grainsize, void *task_dup);
2298 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2299 CGM.IntTy,
2300 CGM.VoidPtrTy,
2301 CGM.IntTy,
2302 CGM.Int64Ty->getPointerTo(),
2303 CGM.Int64Ty->getPointerTo(),
2304 CGM.Int64Ty,
2305 CGM.IntTy,
2306 CGM.IntTy,
2307 CGM.Int64Ty,
2308 CGM.VoidPtrTy};
2309 auto *FnTy =
2310 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2311 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2312 break;
2313 }
2314 case OMPRTL__kmpc_doacross_init: {
2315 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2316 // num_dims, struct kmp_dim *dims);
2317 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2318 CGM.Int32Ty,
2319 CGM.Int32Ty,
2320 CGM.VoidPtrTy};
2321 auto *FnTy =
2322 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2323 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2324 break;
2325 }
2326 case OMPRTL__kmpc_doacross_fini: {
2327 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2328 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2329 auto *FnTy =
2330 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2331 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2332 break;
2333 }
2334 case OMPRTL__kmpc_doacross_post: {
2335 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2336 // *vec);
2337 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2338 CGM.Int64Ty->getPointerTo()};
2339 auto *FnTy =
2340 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2341 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2342 break;
2343 }
2344 case OMPRTL__kmpc_doacross_wait: {
2345 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2346 // *vec);
2347 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2348 CGM.Int64Ty->getPointerTo()};
2349 auto *FnTy =
2350 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2351 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2352 break;
2353 }
2354 case OMPRTL__kmpc_task_reduction_init: {
2355 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2356 // *data);
2357 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2358 auto *FnTy =
2359 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2360 RTLFn =
2361 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2362 break;
2363 }
2364 case OMPRTL__kmpc_task_reduction_get_th_data: {
2365 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2366 // *d);
2367 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2368 auto *FnTy =
2369 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2370 RTLFn = CGM.CreateRuntimeFunction(
2371 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2372 break;
2373 }
2374 case OMPRTL__kmpc_alloc: {
2375 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2376 // al); omp_allocator_handle_t type is void *.
2377 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2378 auto *FnTy =
2379 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2380 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2381 break;
2382 }
2383 case OMPRTL__kmpc_free: {
2384 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2385 // al); omp_allocator_handle_t type is void *.
2386 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2387 auto *FnTy =
2388 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2389 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2390 break;
2391 }
2392 case OMPRTL__kmpc_push_target_tripcount: {
2393 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2394 // size);
2395 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2396 llvm::FunctionType *FnTy =
2397 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2398 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2399 break;
2400 }
2401 case OMPRTL__tgt_target: {
2402 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2403 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2404 // *arg_types);
2405 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2406 CGM.VoidPtrTy,
2407 CGM.Int32Ty,
2408 CGM.VoidPtrPtrTy,
2409 CGM.VoidPtrPtrTy,
2410 CGM.Int64Ty->getPointerTo(),
2411 CGM.Int64Ty->getPointerTo()};
2412 auto *FnTy =
2413 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2414 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2415 break;
2416 }
2417 case OMPRTL__tgt_target_nowait: {
2418 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2419 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2420 // int64_t *arg_types);
2421 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2422 CGM.VoidPtrTy,
2423 CGM.Int32Ty,
2424 CGM.VoidPtrPtrTy,
2425 CGM.VoidPtrPtrTy,
2426 CGM.Int64Ty->getPointerTo(),
2427 CGM.Int64Ty->getPointerTo()};
2428 auto *FnTy =
2429 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2430 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2431 break;
2432 }
2433 case OMPRTL__tgt_target_teams: {
2434 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2435 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2436 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2437 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2438 CGM.VoidPtrTy,
2439 CGM.Int32Ty,
2440 CGM.VoidPtrPtrTy,
2441 CGM.VoidPtrPtrTy,
2442 CGM.Int64Ty->getPointerTo(),
2443 CGM.Int64Ty->getPointerTo(),
2444 CGM.Int32Ty,
2445 CGM.Int32Ty};
2446 auto *FnTy =
2447 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2448 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2449 break;
2450 }
2451 case OMPRTL__tgt_target_teams_nowait: {
2452 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2453 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2454 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2455 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2456 CGM.VoidPtrTy,
2457 CGM.Int32Ty,
2458 CGM.VoidPtrPtrTy,
2459 CGM.VoidPtrPtrTy,
2460 CGM.Int64Ty->getPointerTo(),
2461 CGM.Int64Ty->getPointerTo(),
2462 CGM.Int32Ty,
2463 CGM.Int32Ty};
2464 auto *FnTy =
2465 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2466 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2467 break;
2468 }
2469 case OMPRTL__tgt_register_requires: {
2470 // Build void __tgt_register_requires(int64_t flags);
2471 llvm::Type *TypeParams[] = {CGM.Int64Ty};
2472 auto *FnTy =
2473 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2474 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2475 break;
2476 }
2477 case OMPRTL__tgt_target_data_begin: {
2478 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2479 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2480 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2481 CGM.Int32Ty,
2482 CGM.VoidPtrPtrTy,
2483 CGM.VoidPtrPtrTy,
2484 CGM.Int64Ty->getPointerTo(),
2485 CGM.Int64Ty->getPointerTo()};
2486 auto *FnTy =
2487 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2488 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2489 break;
2490 }
2491 case OMPRTL__tgt_target_data_begin_nowait: {
2492 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2493 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2494 // *arg_types);
2495 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2496 CGM.Int32Ty,
2497 CGM.VoidPtrPtrTy,
2498 CGM.VoidPtrPtrTy,
2499 CGM.Int64Ty->getPointerTo(),
2500 CGM.Int64Ty->getPointerTo()};
2501 auto *FnTy =
2502 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2503 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2504 break;
2505 }
2506 case OMPRTL__tgt_target_data_end: {
2507 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2508 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2509 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2510 CGM.Int32Ty,
2511 CGM.VoidPtrPtrTy,
2512 CGM.VoidPtrPtrTy,
2513 CGM.Int64Ty->getPointerTo(),
2514 CGM.Int64Ty->getPointerTo()};
2515 auto *FnTy =
2516 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2517 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2518 break;
2519 }
2520 case OMPRTL__tgt_target_data_end_nowait: {
2521 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2522 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2523 // *arg_types);
2524 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2525 CGM.Int32Ty,
2526 CGM.VoidPtrPtrTy,
2527 CGM.VoidPtrPtrTy,
2528 CGM.Int64Ty->getPointerTo(),
2529 CGM.Int64Ty->getPointerTo()};
2530 auto *FnTy =
2531 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2532 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2533 break;
2534 }
2535 case OMPRTL__tgt_target_data_update: {
2536 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2537 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2538 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2539 CGM.Int32Ty,
2540 CGM.VoidPtrPtrTy,
2541 CGM.VoidPtrPtrTy,
2542 CGM.Int64Ty->getPointerTo(),
2543 CGM.Int64Ty->getPointerTo()};
2544 auto *FnTy =
2545 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2546 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2547 break;
2548 }
2549 case OMPRTL__tgt_target_data_update_nowait: {
2550 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2551 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2552 // *arg_types);
2553 llvm::Type *TypeParams[] = {CGM.Int64Ty,
2554 CGM.Int32Ty,
2555 CGM.VoidPtrPtrTy,
2556 CGM.VoidPtrPtrTy,
2557 CGM.Int64Ty->getPointerTo(),
2558 CGM.Int64Ty->getPointerTo()};
2559 auto *FnTy =
2560 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2561 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2562 break;
2563 }
2564 case OMPRTL__tgt_mapper_num_components: {
2565 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2566 llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2567 auto *FnTy =
2568 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2569 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2570 break;
2571 }
2572 case OMPRTL__tgt_push_mapper_component: {
2573 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2574 // *base, void *begin, int64_t size, int64_t type);
2575 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2576 CGM.Int64Ty, CGM.Int64Ty};
2577 auto *FnTy =
2578 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2579 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2580 break;
2581 }
2582 }
2583 assert(RTLFn && "Unable to find OpenMP runtime function")((RTLFn && "Unable to find OpenMP runtime function") ?
static_cast<void> (0) : __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2583, __PRETTY_FUNCTION__))
;
2584 return RTLFn;
2585}
2586
2587llvm::FunctionCallee
2588CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2589 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2590, __PRETTY_FUNCTION__))
2590 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2590, __PRETTY_FUNCTION__))
;
2591 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2592 : "__kmpc_for_static_init_4u")
2593 : (IVSigned ? "__kmpc_for_static_init_8"
2594 : "__kmpc_for_static_init_8u");
2595 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2596 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2597 llvm::Type *TypeParams[] = {
2598 getIdentTyPointerTy(), // loc
2599 CGM.Int32Ty, // tid
2600 CGM.Int32Ty, // schedtype
2601 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2602 PtrTy, // p_lower
2603 PtrTy, // p_upper
2604 PtrTy, // p_stride
2605 ITy, // incr
2606 ITy // chunk
2607 };
2608 auto *FnTy =
2609 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2610 return CGM.CreateRuntimeFunction(FnTy, Name);
2611}
2612
2613llvm::FunctionCallee
2614CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2615 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2616, __PRETTY_FUNCTION__))
2616 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2616, __PRETTY_FUNCTION__))
;
2617 StringRef Name =
2618 IVSize == 32
2619 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2620 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2621 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2622 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2623 CGM.Int32Ty, // tid
2624 CGM.Int32Ty, // schedtype
2625 ITy, // lower
2626 ITy, // upper
2627 ITy, // stride
2628 ITy // chunk
2629 };
2630 auto *FnTy =
2631 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2632 return CGM.CreateRuntimeFunction(FnTy, Name);
2633}
2634
2635llvm::FunctionCallee
2636CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2637 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2638, __PRETTY_FUNCTION__))
2638 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2638, __PRETTY_FUNCTION__))
;
2639 StringRef Name =
2640 IVSize == 32
2641 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2642 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2643 llvm::Type *TypeParams[] = {
2644 getIdentTyPointerTy(), // loc
2645 CGM.Int32Ty, // tid
2646 };
2647 auto *FnTy =
2648 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2649 return CGM.CreateRuntimeFunction(FnTy, Name);
2650}
2651
2652llvm::FunctionCallee
2653CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2654 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2655, __PRETTY_FUNCTION__))
2655 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2655, __PRETTY_FUNCTION__))
;
2656 StringRef Name =
2657 IVSize == 32
2658 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2659 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2660 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2661 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2662 llvm::Type *TypeParams[] = {
2663 getIdentTyPointerTy(), // loc
2664 CGM.Int32Ty, // tid
2665 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2666 PtrTy, // p_lower
2667 PtrTy, // p_upper
2668 PtrTy // p_stride
2669 };
2670 auto *FnTy =
2671 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2672 return CGM.CreateRuntimeFunction(FnTy, Name);
2673}
2674
2675/// Obtain information that uniquely identifies a target entry. This
2676/// consists of the file and device IDs as well as line number associated with
2677/// the relevant entry source location.
2678static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2679 unsigned &DeviceID, unsigned &FileID,
2680 unsigned &LineNum) {
2681 SourceManager &SM = C.getSourceManager();
2682
2683 // The loc should be always valid and have a file ID (the user cannot use
2684 // #pragma directives in macros)
2685
2686 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2686, __PRETTY_FUNCTION__))
;
2687
2688 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2689 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2689, __PRETTY_FUNCTION__))
;
2690
2691 llvm::sys::fs::UniqueID ID;
2692 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2693 SM.getDiagnostics().Report(diag::err_cannot_open_file)
2694 << PLoc.getFilename() << EC.message();
2695
2696 DeviceID = ID.getDevice();
2697 FileID = ID.getFile();
2698 LineNum = PLoc.getLine();
2699}
2700
2701Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2702 if (CGM.getLangOpts().OpenMPSimd)
2703 return Address::invalid();
2704 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2705 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2706 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2707 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2708 HasRequiresUnifiedSharedMemory))) {
2709 SmallString<64> PtrName;
2710 {
2711 llvm::raw_svector_ostream OS(PtrName);
2712 OS << CGM.getMangledName(GlobalDecl(VD));
2713 if (!VD->isExternallyVisible()) {
2714 unsigned DeviceID, FileID, Line;
2715 getTargetEntryUniqueInfo(CGM.getContext(),
2716 VD->getCanonicalDecl()->getBeginLoc(),
2717 DeviceID, FileID, Line);
2718 OS << llvm::format("_%x", FileID);
2719 }
2720 OS << "_decl_tgt_ref_ptr";
2721 }
2722 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2723 if (!Ptr) {
2724 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2725 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2726 PtrName);
2727
2728 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2729 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2730
2731 if (!CGM.getLangOpts().OpenMPIsDevice)
2732 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2733 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2734 }
2735 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2736 }
2737 return Address::invalid();
2738}
2739
2740llvm::Constant *
2741CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2742 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2743, __PRETTY_FUNCTION__))
2743 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2743, __PRETTY_FUNCTION__))
;
2744 // Lookup the entry, lazily creating it if necessary.
2745 std::string Suffix = getName({"cache", ""});
2746 return getOrCreateInternalVariable(
2747 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2748}
2749
2750Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2751 const VarDecl *VD,
2752 Address VDAddr,
2753 SourceLocation Loc) {
2754 if (CGM.getLangOpts().OpenMPUseTLS &&
2755 CGM.getContext().getTargetInfo().isTLSSupported())
2756 return VDAddr;
2757
2758 llvm::Type *VarTy = VDAddr.getElementType();
2759 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2760 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2761 CGM.Int8PtrTy),
2762 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2763 getOrCreateThreadPrivateCache(VD)};
2764 return Address(CGF.EmitRuntimeCall(
2765 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2766 VDAddr.getAlignment());
2767}
2768
2769void CGOpenMPRuntime::emitThreadPrivateVarInit(
2770 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2771 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2772 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2773 // library.
2774 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2775 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2776 OMPLoc);
2777 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2778 // to register constructor/destructor for variable.
2779 llvm::Value *Args[] = {
2780 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2781 Ctor, CopyCtor, Dtor};
2782 CGF.EmitRuntimeCall(
2783 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2784}
2785
2786llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2787 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2788 bool PerformInit, CodeGenFunction *CGF) {
2789 if (CGM.getLangOpts().OpenMPUseTLS &&
2790 CGM.getContext().getTargetInfo().isTLSSupported())
2791 return nullptr;
2792
2793 VD = VD->getDefinition(CGM.getContext());
2794 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2795 QualType ASTTy = VD->getType();
2796
2797 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2798 const Expr *Init = VD->getAnyInitializer();
2799 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2800 // Generate function that re-emits the declaration's initializer into the
2801 // threadprivate copy of the variable VD
2802 CodeGenFunction CtorCGF(CGM);
2803 FunctionArgList Args;
2804 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2805 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2806 ImplicitParamDecl::Other);
2807 Args.push_back(&Dst);
2808
2809 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2810 CGM.getContext().VoidPtrTy, Args);
2811 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2812 std::string Name = getName({"__kmpc_global_ctor_", ""});
2813 llvm::Function *Fn =
2814 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2815 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2816 Args, Loc, Loc);
2817 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2818 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2819 CGM.getContext().VoidPtrTy, Dst.getLocation());
2820 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2821 Arg = CtorCGF.Builder.CreateElementBitCast(
2822 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2823 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2824 /*IsInitializer=*/true);
2825 ArgVal = CtorCGF.EmitLoadOfScalar(
2826 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2827 CGM.getContext().VoidPtrTy, Dst.getLocation());
2828 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2829 CtorCGF.FinishFunction();
2830 Ctor = Fn;
2831 }
2832 if (VD->getType().isDestructedType() != QualType::DK_none) {
2833 // Generate function that emits destructor call for the threadprivate copy
2834 // of the variable VD
2835 CodeGenFunction DtorCGF(CGM);
2836 FunctionArgList Args;
2837 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2838 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2839 ImplicitParamDecl::Other);
2840 Args.push_back(&Dst);
2841
2842 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2843 CGM.getContext().VoidTy, Args);
2844 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2845 std::string Name = getName({"__kmpc_global_dtor_", ""});
2846 llvm::Function *Fn =
2847 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2848 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2849 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2850 Loc, Loc);
2851 // Create a scope with an artificial location for the body of this function.
2852 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2853 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2854 DtorCGF.GetAddrOfLocalVar(&Dst),
2855 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2856 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2857 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2858 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2859 DtorCGF.FinishFunction();
2860 Dtor = Fn;
2861 }
2862 // Do not emit init function if it is not required.
2863 if (!Ctor && !Dtor)
2864 return nullptr;
2865
2866 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2867 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2868 /*isVarArg=*/false)
2869 ->getPointerTo();
2870 // Copying constructor for the threadprivate variable.
2871 // Must be NULL - reserved by runtime, but currently it requires that this
2872 // parameter is always NULL. Otherwise it fires assertion.
2873 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2874 if (Ctor == nullptr) {
2875 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2876 /*isVarArg=*/false)
2877 ->getPointerTo();
2878 Ctor = llvm::Constant::getNullValue(CtorTy);
2879 }
2880 if (Dtor == nullptr) {
2881 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2882 /*isVarArg=*/false)
2883 ->getPointerTo();
2884 Dtor = llvm::Constant::getNullValue(DtorTy);
2885 }
2886 if (!CGF) {
2887 auto *InitFunctionTy =
2888 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2889 std::string Name = getName({"__omp_threadprivate_init_", ""});
2890 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2891 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2892 CodeGenFunction InitCGF(CGM);
2893 FunctionArgList ArgList;
2894 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2895 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2896 Loc, Loc);
2897 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2898 InitCGF.FinishFunction();
2899 return InitFunction;
2900 }
2901 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2902 }
2903 return nullptr;
2904}
2905
2906bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2907 llvm::GlobalVariable *Addr,
2908 bool PerformInit) {
2909 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2910 !CGM.getLangOpts().OpenMPIsDevice)
2911 return false;
2912 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2913 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2914 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2915 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2916 HasRequiresUnifiedSharedMemory))
2917 return CGM.getLangOpts().OpenMPIsDevice;
2918 VD = VD->getDefinition(CGM.getContext());
2919 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2920 return CGM.getLangOpts().OpenMPIsDevice;
2921
2922 QualType ASTTy = VD->getType();
2923
2924 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2925 // Produce the unique prefix to identify the new target regions. We use
2926 // the source location of the variable declaration which we know to not
2927 // conflict with any target region.
2928 unsigned DeviceID;
2929 unsigned FileID;
2930 unsigned Line;
2931 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2932 SmallString<128> Buffer, Out;
2933 {
2934 llvm::raw_svector_ostream OS(Buffer);
2935 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2936 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2937 }
2938
2939 const Expr *Init = VD->getAnyInitializer();
2940 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2941 llvm::Constant *Ctor;
2942 llvm::Constant *ID;
2943 if (CGM.getLangOpts().OpenMPIsDevice) {
2944 // Generate function that re-emits the declaration's initializer into
2945 // the threadprivate copy of the variable VD
2946 CodeGenFunction CtorCGF(CGM);
2947
2948 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2949 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2950 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2951 FTy, Twine(Buffer, "_ctor"), FI, Loc);
2952 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2953 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2954 FunctionArgList(), Loc, Loc);
2955 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2956 CtorCGF.EmitAnyExprToMem(Init,
2957 Address(Addr, CGM.getContext().getDeclAlign(VD)),
2958 Init->getType().getQualifiers(),
2959 /*IsInitializer=*/true);
2960 CtorCGF.FinishFunction();
2961 Ctor = Fn;
2962 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2963 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2964 } else {
2965 Ctor = new llvm::GlobalVariable(
2966 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2967 llvm::GlobalValue::PrivateLinkage,
2968 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2969 ID = Ctor;
2970 }
2971
2972 // Register the information for the entry associated with the constructor.
2973 Out.clear();
2974 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2975 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2976 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2977 }
2978 if (VD->getType().isDestructedType() != QualType::DK_none) {
2979 llvm::Constant *Dtor;
2980 llvm::Constant *ID;
2981 if (CGM.getLangOpts().OpenMPIsDevice) {
2982 // Generate function that emits destructor call for the threadprivate
2983 // copy of the variable VD
2984 CodeGenFunction DtorCGF(CGM);
2985
2986 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2987 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2988 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2989 FTy, Twine(Buffer, "_dtor"), FI, Loc);
2990 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2991 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2992 FunctionArgList(), Loc, Loc);
2993 // Create a scope with an artificial location for the body of this
2994 // function.
2995 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2996 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2997 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2998 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2999 DtorCGF.FinishFunction();
3000 Dtor = Fn;
3001 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
3002 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
3003 } else {
3004 Dtor = new llvm::GlobalVariable(
3005 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
3006 llvm::GlobalValue::PrivateLinkage,
3007 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
3008 ID = Dtor;
3009 }
3010 // Register the information for the entry associated with the destructor.
3011 Out.clear();
3012 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
3013 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
3014 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
3015 }
3016 return CGM.getLangOpts().OpenMPIsDevice;
3017}
3018
3019Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
3020 QualType VarType,
3021 StringRef Name) {
3022 std::string Suffix = getName({"artificial", ""});
3023 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
3024 llvm::Value *GAddr =
3025 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
3026 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
3027 CGM.getTarget().isTLSSupported()) {
3028 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
3029 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
3030 }
3031 std::string CacheSuffix = getName({"cache", ""});
3032 llvm::Value *Args[] = {
3033 emitUpdateLocation(CGF, SourceLocation()),
3034 getThreadID(CGF, SourceLocation()),
3035 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
3036 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
3037 /*isSigned=*/false),
3038 getOrCreateInternalVariable(
3039 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
3040 return Address(
3041 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3042 CGF.EmitRuntimeCall(
3043 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3044 VarLVType->getPointerTo(/*AddrSpace=*/0)),
3045 CGM.getContext().getTypeAlignInChars(VarType));
3046}
3047
3048void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
3049 const RegionCodeGenTy &ThenGen,
3050 const RegionCodeGenTy &ElseGen) {
3051 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3052
3053 // If the condition constant folds and can be elided, try to avoid emitting
3054 // the condition and the dead arm of the if/else.
3055 bool CondConstant;
3056 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
9
Assuming the condition is false
10
Taking false branch
3057 if (CondConstant)
3058 ThenGen(CGF);
3059 else
3060 ElseGen(CGF);
3061 return;
3062 }
3063
3064 // Otherwise, the condition did not fold, or we couldn't elide it. Just
3065 // emit the conditional branch.
3066 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3067 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3068 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3069 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3070
3071 // Emit the 'then' code.
3072 CGF.EmitBlock(ThenBlock);
3073 ThenGen(CGF);
11
Calling 'RegionCodeGenTy::operator()'
3074 CGF.EmitBranch(ContBlock);
3075 // Emit the 'else' code if present.
3076 // There is no need to emit line number for unconditional branch.
3077 (void)ApplyDebugLocation::CreateEmpty(CGF);
3078 CGF.EmitBlock(ElseBlock);
3079 ElseGen(CGF);
3080 // There is no need to emit line number for unconditional branch.
3081 (void)ApplyDebugLocation::CreateEmpty(CGF);
3082 CGF.EmitBranch(ContBlock);
3083 // Emit the continuation block for code after the if.
3084 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3085}
3086
3087void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3088 llvm::Function *OutlinedFn,
3089 ArrayRef<llvm::Value *> CapturedVars,
3090 const Expr *IfCond) {
3091 if (!CGF.HaveInsertPoint())
3092 return;
3093 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3094 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3095 PrePostActionTy &) {
3096 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3097 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3098 llvm::Value *Args[] = {
3099 RTLoc,
3100 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3101 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3102 llvm::SmallVector<llvm::Value *, 16> RealArgs;
3103 RealArgs.append(std::begin(Args), std::end(Args));
3104 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3105
3106 llvm::FunctionCallee RTLFn =
3107 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3108 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3109 };
3110 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3111 PrePostActionTy &) {
3112 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3113 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3114 // Build calls:
3115 // __kmpc_serialized_parallel(&Loc, GTid);
3116 llvm::Value *Args[] = {RTLoc, ThreadID};
3117 CGF.EmitRuntimeCall(
3118 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3119
3120 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
3121 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
3122 Address ZeroAddrBound =
3123 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3124 /*Name=*/".bound.zero.addr");
3125 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
3126 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3127 // ThreadId for serialized parallels is 0.
3128 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
3129 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
3130 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3131 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3132
3133 // __kmpc_end_serialized_parallel(&Loc, GTid);
3134 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3135 CGF.EmitRuntimeCall(
3136 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3137 EndArgs);
3138 };
3139 if (IfCond) {
3140 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3141 } else {
3142 RegionCodeGenTy ThenRCG(ThenGen);
3143 ThenRCG(CGF);
3144 }
3145}
3146
3147// If we're inside an (outlined) parallel region, use the region info's
3148// thread-ID variable (it is passed in a first argument of the outlined function
3149// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3150// regular serial code region, get thread ID by calling kmp_int32
3151// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3152// return the address of that temp.
3153Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3154 SourceLocation Loc) {
3155 if (auto *OMPRegionInfo =
3156 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3157 if (OMPRegionInfo->getThreadIDVariable())
3158 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3159
3160 llvm::Value *ThreadID = getThreadID(CGF, Loc);
3161 QualType Int32Ty =
3162 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3163 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3164 CGF.EmitStoreOfScalar(ThreadID,
3165 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3166
3167 return ThreadIDTemp;
3168}
3169
3170llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3171 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3172 SmallString<256> Buffer;
3173 llvm::raw_svector_ostream Out(Buffer);
3174 Out << Name;
3175 StringRef RuntimeName = Out.str();
3176 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3177 if (Elem.second) {
3178 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3179, __PRETTY_FUNCTION__))
3179 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3179, __PRETTY_FUNCTION__))
;
3180 return &*Elem.second;
3181 }
3182
3183 return Elem.second = new llvm::GlobalVariable(
3184 CGM.getModule(), Ty, /*IsConstant*/ false,
3185 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3186 Elem.first(), /*InsertBefore=*/nullptr,
3187 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3188}
3189
3190llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3191 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3192 std::string Name = getName({Prefix, "var"});
3193 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3194}
3195
3196namespace {
3197/// Common pre(post)-action for different OpenMP constructs.
3198class CommonActionTy final : public PrePostActionTy {
3199 llvm::FunctionCallee EnterCallee;
3200 ArrayRef<llvm::Value *> EnterArgs;
3201 llvm::FunctionCallee ExitCallee;
3202 ArrayRef<llvm::Value *> ExitArgs;
3203 bool Conditional;
3204 llvm::BasicBlock *ContBlock = nullptr;
3205
3206public:
3207 CommonActionTy(llvm::FunctionCallee EnterCallee,
3208 ArrayRef<llvm::Value *> EnterArgs,
3209 llvm::FunctionCallee ExitCallee,
3210 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3211 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3212 ExitArgs(ExitArgs), Conditional(Conditional) {}
3213 void Enter(CodeGenFunction &CGF) override {
3214 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3215 if (Conditional) {
3216 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3217 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3218 ContBlock = CGF.createBasicBlock("omp_if.end");
3219 // Generate the branch (If-stmt)
3220 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3221 CGF.EmitBlock(ThenBlock);
3222 }
3223 }
3224 void Done(CodeGenFunction &CGF) {
3225 // Emit the rest of blocks/branches
3226 CGF.EmitBranch(ContBlock);
3227 CGF.EmitBlock(ContBlock, true);
3228 }
3229 void Exit(CodeGenFunction &CGF) override {
3230 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3231 }
3232};
3233} // anonymous namespace
3234
3235void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3236 StringRef CriticalName,
3237 const RegionCodeGenTy &CriticalOpGen,
3238 SourceLocation Loc, const Expr *Hint) {
3239 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3240 // CriticalOpGen();
3241 // __kmpc_end_critical(ident_t *, gtid, Lock);
3242 // Prepare arguments and build a call to __kmpc_critical
3243 if (!CGF.HaveInsertPoint())
3244 return;
3245 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3246 getCriticalRegionLock(CriticalName)};
3247 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3248 std::end(Args));
3249 if (Hint) {
3250 EnterArgs.push_back(CGF.Builder.CreateIntCast(
3251 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3252 }
3253 CommonActionTy Action(
3254 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3255 : OMPRTL__kmpc_critical),
3256 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3257 CriticalOpGen.setAction(Action);
3258 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3259}
3260
3261void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3262 const RegionCodeGenTy &MasterOpGen,
3263 SourceLocation Loc) {
3264 if (!CGF.HaveInsertPoint())
3265 return;
3266 // if(__kmpc_master(ident_t *, gtid)) {
3267 // MasterOpGen();
3268 // __kmpc_end_master(ident_t *, gtid);
3269 // }
3270 // Prepare arguments and build a call to __kmpc_master
3271 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3272 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3273 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3274 /*Conditional=*/true);
3275 MasterOpGen.setAction(Action);
3276 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3277 Action.Done(CGF);
3278}
3279
3280void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3281 SourceLocation Loc) {
3282 if (!CGF.HaveInsertPoint())
3283 return;
3284 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3285 llvm::Value *Args[] = {
3286 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3287 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3288 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3289 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3290 Region->emitUntiedSwitch(CGF);
3291}
3292
3293void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3294 const RegionCodeGenTy &TaskgroupOpGen,
3295 SourceLocation Loc) {
3296 if (!CGF.HaveInsertPoint())
3297 return;
3298 // __kmpc_taskgroup(ident_t *, gtid);
3299 // TaskgroupOpGen();
3300 // __kmpc_end_taskgroup(ident_t *, gtid);
3301 // Prepare arguments and build a call to __kmpc_taskgroup
3302 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3303 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3304 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3305 Args);
3306 TaskgroupOpGen.setAction(Action);
3307 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3308}
3309
3310/// Given an array of pointers to variables, project the address of a
3311/// given variable.
3312static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3313 unsigned Index, const VarDecl *Var) {
3314 // Pull out the pointer to the variable.
3315 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3316 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3317
3318 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3319 Addr = CGF.Builder.CreateElementBitCast(
3320 Addr, CGF.ConvertTypeForMem(Var->getType()));
3321 return Addr;
3322}
3323
3324static llvm::Value *emitCopyprivateCopyFunction(
3325 CodeGenModule &CGM, llvm::Type *ArgsType,
3326 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3327 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3328 SourceLocation Loc) {
3329 ASTContext &C = CGM.getContext();
3330 // void copy_func(void *LHSArg, void *RHSArg);
3331 FunctionArgList Args;
3332 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3333 ImplicitParamDecl::Other);
3334 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3335 ImplicitParamDecl::Other);
3336 Args.push_back(&LHSArg);
3337 Args.push_back(&RHSArg);
3338 const auto &CGFI =
3339 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3340 std::string Name =
3341 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3342 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3343 llvm::GlobalValue::InternalLinkage, Name,
3344 &CGM.getModule());
3345 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3346 Fn->setDoesNotRecurse();
3347 CodeGenFunction CGF(CGM);
3348 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3349 // Dest = (void*[n])(LHSArg);
3350 // Src = (void*[n])(RHSArg);
3351 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3352 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3353 ArgsType), CGF.getPointerAlign());
3354 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3355 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3356 ArgsType), CGF.getPointerAlign());
3357 // *(Type0*)Dst[0] = *(Type0*)Src[0];
3358 // *(Type1*)Dst[1] = *(Type1*)Src[1];
3359 // ...
3360 // *(Typen*)Dst[n] = *(Typen*)Src[n];
3361 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3362 const auto *DestVar =
3363 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3364 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3365
3366 const auto *SrcVar =
3367 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3368 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3369
3370 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3371 QualType Type = VD->getType();
3372 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3373 }
3374 CGF.FinishFunction();
3375 return Fn;
3376}
3377
3378void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3379 const RegionCodeGenTy &SingleOpGen,
3380 SourceLocation Loc,
3381 ArrayRef<const Expr *> CopyprivateVars,
3382 ArrayRef<const Expr *> SrcExprs,
3383 ArrayRef<const Expr *> DstExprs,
3384 ArrayRef<const Expr *> AssignmentOps) {
3385 if (!CGF.HaveInsertPoint())
3386 return;
3387 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3389, __PRETTY_FUNCTION__))
3388 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3389, __PRETTY_FUNCTION__))
3389 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3389, __PRETTY_FUNCTION__))
;
3390 ASTContext &C = CGM.getContext();
3391 // int32 did_it = 0;
3392 // if(__kmpc_single(ident_t *, gtid)) {
3393 // SingleOpGen();
3394 // __kmpc_end_single(ident_t *, gtid);
3395 // did_it = 1;
3396 // }
3397 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3398 // <copy_func>, did_it);
3399
3400 Address DidIt = Address::invalid();
3401 if (!CopyprivateVars.empty()) {
3402 // int32 did_it = 0;
3403 QualType KmpInt32Ty =
3404 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3405 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3406 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3407 }
3408 // Prepare arguments and build a call to __kmpc_single
3409 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3410 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3411 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3412 /*Conditional=*/true);
3413 SingleOpGen.setAction(Action);
3414 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3415 if (DidIt.isValid()) {
3416 // did_it = 1;
3417 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3418 }
3419 Action.Done(CGF);
3420 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3421 // <copy_func>, did_it);
3422 if (DidIt.isValid()) {
3423 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3424 QualType CopyprivateArrayTy = C.getConstantArrayType(
3425 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3426 /*IndexTypeQuals=*/0);
3427 // Create a list of all private variables for copyprivate.
3428 Address CopyprivateList =
3429 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3430 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3431 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3432 CGF.Builder.CreateStore(
3433 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3434 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
3435 CGF.VoidPtrTy),
3436 Elem);
3437 }
3438 // Build function that copies private values from single region to all other
3439 // threads in the corresponding parallel region.
3440 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3441 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3442 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3443 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3444 Address CL =
3445 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3446 CGF.VoidPtrTy);
3447 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3448 llvm::Value *Args[] = {
3449 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3450 getThreadID(CGF, Loc), // i32 <gtid>
3451 BufSize, // size_t <buf_size>
3452 CL.getPointer(), // void *<copyprivate list>
3453 CpyFn, // void (*) (void *, void *) <copy_func>
3454 DidItVal // i32 did_it
3455 };
3456 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3457 }
3458}
3459
3460void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3461 const RegionCodeGenTy &OrderedOpGen,
3462 SourceLocation Loc, bool IsThreads) {
3463 if (!CGF.HaveInsertPoint())
3464 return;
3465 // __kmpc_ordered(ident_t *, gtid);
3466 // OrderedOpGen();
3467 // __kmpc_end_ordered(ident_t *, gtid);
3468 // Prepare arguments and build a call to __kmpc_ordered
3469 if (IsThreads) {
3470 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3471 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3472 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3473 Args);
3474 OrderedOpGen.setAction(Action);
3475 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3476 return;
3477 }
3478 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3479}
3480
3481unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3482 unsigned Flags;
3483 if (Kind == OMPD_for)
3484 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3485 else if (Kind == OMPD_sections)
3486 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3487 else if (Kind == OMPD_single)
3488 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3489 else if (Kind == OMPD_barrier)
3490 Flags = OMP_IDENT_BARRIER_EXPL;
3491 else
3492 Flags = OMP_IDENT_BARRIER_IMPL;
3493 return Flags;
3494}
3495
3496void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3497 CodeGenFunction &CGF, const OMPLoopDirective &S,
3498 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3499 // Check if the loop directive is actually a doacross loop directive. In this
3500 // case choose static, 1 schedule.
3501 if (llvm::any_of(
3502 S.getClausesOfKind<OMPOrderedClause>(),
3503 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3504 ScheduleKind = OMPC_SCHEDULE_static;
3505 // Chunk size is 1 in this case.
3506 llvm::APInt ChunkSize(32, 1);
3507 ChunkExpr = IntegerLiteral::Create(
3508 CGF.getContext(), ChunkSize,
3509 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3510 SourceLocation());
3511 }
3512}
3513
3514void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3515 OpenMPDirectiveKind Kind, bool EmitChecks,
3516 bool ForceSimpleCall) {
3517 // Check if we should use the OMPBuilder
3518 auto *OMPRegionInfo =
3519 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
3520 llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3521 if (OMPBuilder) {
3522 CGF.Builder.restoreIP(OMPBuilder->CreateBarrier(
3523 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
3524 return;
3525 }
3526
3527 if (!CGF.HaveInsertPoint())
3528 return;
3529 // Build call __kmpc_cancel_barrier(loc, thread_id);
3530 // Build call __kmpc_barrier(loc, thread_id);
3531 unsigned Flags = getDefaultFlagsForBarriers(Kind);
3532 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3533 // thread_id);
3534 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3535 getThreadID(CGF, Loc)};
3536 if (OMPRegionInfo) {
3537 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3538 llvm::Value *Result = CGF.EmitRuntimeCall(
3539 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3540 if (EmitChecks) {
3541 // if (__kmpc_cancel_barrier()) {
3542 // exit from construct;
3543 // }
3544 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3545 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3546 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3547 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3548 CGF.EmitBlock(ExitBB);
3549 // exit from construct;
3550 CodeGenFunction::JumpDest CancelDestination =
3551 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3552 CGF.EmitBranchThroughCleanup(CancelDestination);
3553 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3554 }
3555 return;
3556 }
3557 }
3558 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3559}
3560
3561/// Map the OpenMP loop schedule to the runtime enumeration.
3562static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3563 bool Chunked, bool Ordered) {
3564 switch (ScheduleKind) {
3565 case OMPC_SCHEDULE_static:
3566 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3567 : (Ordered ? OMP_ord_static : OMP_sch_static);
3568 case OMPC_SCHEDULE_dynamic:
3569 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3570 case OMPC_SCHEDULE_guided:
3571 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3572 case OMPC_SCHEDULE_runtime:
3573 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3574 case OMPC_SCHEDULE_auto:
3575 return Ordered ? OMP_ord_auto : OMP_sch_auto;
3576 case OMPC_SCHEDULE_unknown:
3577 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3577, __PRETTY_FUNCTION__))
;
3578 return Ordered ? OMP_ord_static : OMP_sch_static;
3579 }
3580 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3580)
;
3581}
3582
3583/// Map the OpenMP distribute schedule to the runtime enumeration.
3584static OpenMPSchedType
3585getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3586 // only static is allowed for dist_schedule
3587 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3588}
3589
3590bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3591 bool Chunked) const {
3592 OpenMPSchedType Schedule =
3593 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3594 return Schedule == OMP_sch_static;
3595}
3596
3597bool CGOpenMPRuntime::isStaticNonchunked(
3598 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3599 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3600 return Schedule == OMP_dist_sch_static;
3601}
3602
3603bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3604 bool Chunked) const {
3605 OpenMPSchedType Schedule =
3606 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3607 return Schedule == OMP_sch_static_chunked;
3608}
3609
3610bool CGOpenMPRuntime::isStaticChunked(
3611 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3612 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3613 return Schedule == OMP_dist_sch_static_chunked;
3614}
3615
3616bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3617 OpenMPSchedType Schedule =
3618 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3619 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3619, __PRETTY_FUNCTION__))
;
3620 return Schedule != OMP_sch_static;
3621}
3622
3623static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3624 OpenMPScheduleClauseModifier M1,
3625 OpenMPScheduleClauseModifier M2) {
3626 int Modifier = 0;
3627 switch (M1) {
3628 case OMPC_SCHEDULE_MODIFIER_monotonic:
3629 Modifier = OMP_sch_modifier_monotonic;
3630 break;
3631 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3632 Modifier = OMP_sch_modifier_nonmonotonic;
3633 break;
3634 case OMPC_SCHEDULE_MODIFIER_simd:
3635 if (Schedule == OMP_sch_static_chunked)
3636 Schedule = OMP_sch_static_balanced_chunked;
3637 break;
3638 case OMPC_SCHEDULE_MODIFIER_last:
3639 case OMPC_SCHEDULE_MODIFIER_unknown:
3640 break;
3641 }
3642 switch (M2) {
3643 case OMPC_SCHEDULE_MODIFIER_monotonic:
3644 Modifier = OMP_sch_modifier_monotonic;
3645 break;
3646 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3647 Modifier = OMP_sch_modifier_nonmonotonic;
3648 break;
3649 case OMPC_SCHEDULE_MODIFIER_simd:
3650 if (Schedule == OMP_sch_static_chunked)
3651 Schedule = OMP_sch_static_balanced_chunked;
3652 break;
3653 case OMPC_SCHEDULE_MODIFIER_last:
3654 case OMPC_SCHEDULE_MODIFIER_unknown:
3655 break;
3656 }
3657 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3658 // If the static schedule kind is specified or if the ordered clause is
3659 // specified, and if the nonmonotonic modifier is not specified, the effect is
3660 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3661 // modifier is specified, the effect is as if the nonmonotonic modifier is
3662 // specified.
3663 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3664 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3665 Schedule == OMP_sch_static_balanced_chunked ||
3666 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
3667 Schedule == OMP_dist_sch_static_chunked ||
3668 Schedule == OMP_dist_sch_static))
3669 Modifier = OMP_sch_modifier_nonmonotonic;
3670 }
3671 return Schedule | Modifier;
3672}
3673
3674void CGOpenMPRuntime::emitForDispatchInit(
3675 CodeGenFunction &CGF, SourceLocation Loc,
3676 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3677 bool Ordered, const DispatchRTInput &DispatchValues) {
3678 if (!CGF.HaveInsertPoint())
3679 return;
3680 OpenMPSchedType Schedule = getRuntimeSchedule(
3681 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3682 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3685, __PRETTY_FUNCTION__))
3683 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3685, __PRETTY_FUNCTION__))
3684 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3685, __PRETTY_FUNCTION__))
3685 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3685, __PRETTY_FUNCTION__))
;
3686 // Call __kmpc_dispatch_init(
3687 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3688 // kmp_int[32|64] lower, kmp_int[32|64] upper,
3689 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
3690
3691 // If the Chunk was not specified in the clause - use default value 1.
3692 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3693 : CGF.Builder.getIntN(IVSize, 1);
3694 llvm::Value *Args[] = {
3695 emitUpdateLocation(CGF, Loc),
3696 getThreadID(CGF, Loc),
3697 CGF.Builder.getInt32(addMonoNonMonoModifier(
3698 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3699 DispatchValues.LB, // Lower
3700 DispatchValues.UB, // Upper
3701 CGF.Builder.getIntN(IVSize, 1), // Stride
3702 Chunk // Chunk
3703 };
3704 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3705}
3706
3707static void emitForStaticInitCall(
3708 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3709 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3710 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3711 const CGOpenMPRuntime::StaticRTInput &Values) {
3712 if (!CGF.HaveInsertPoint())
3713 return;
3714
3715 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3715, __PRETTY_FUNCTION__))
;
3716 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3720, __PRETTY_FUNCTION__))
3717 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3720, __PRETTY_FUNCTION__))
3718 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3720, __PRETTY_FUNCTION__))
3719 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3720, __PRETTY_FUNCTION__))
3720 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3720, __PRETTY_FUNCTION__))
;
3721
3722 // Call __kmpc_for_static_init(
3723 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3724 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3725 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3726 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3727 llvm::Value *Chunk = Values.Chunk;
3728 if (Chunk == nullptr) {
3729 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3731, __PRETTY_FUNCTION__))
3730 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3731, __PRETTY_FUNCTION__))
3731 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3731, __PRETTY_FUNCTION__))
;
3732 // If the Chunk was not specified in the clause - use default value 1.
3733 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3734 } else {
3735 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3739, __PRETTY_FUNCTION__))
3736 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3739, __PRETTY_FUNCTION__))
3737 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3739, __PRETTY_FUNCTION__))
3738 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3739, __PRETTY_FUNCTION__))
3739 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3739, __PRETTY_FUNCTION__))
;
3740 }
3741 llvm::Value *Args[] = {
3742 UpdateLocation,
3743 ThreadId,
3744 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3745 M2)), // Schedule type
3746 Values.IL.getPointer(), // &isLastIter
3747 Values.LB.getPointer(), // &LB
3748 Values.UB.getPointer(), // &UB
3749 Values.ST.getPointer(), // &Stride
3750 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3751 Chunk // Chunk
3752 };
3753 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3754}
3755
3756void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3757 SourceLocation Loc,
3758 OpenMPDirectiveKind DKind,
3759 const OpenMPScheduleTy &ScheduleKind,
3760 const StaticRTInput &Values) {
3761 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3762 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3763 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3764, __PRETTY_FUNCTION__))
3764 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3764, __PRETTY_FUNCTION__))
;
3765 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3766 isOpenMPLoopDirective(DKind)
3767 ? OMP_IDENT_WORK_LOOP
3768 : OMP_IDENT_WORK_SECTIONS);
3769 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3770 llvm::FunctionCallee StaticInitFunction =
3771 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3772 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3773 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3774}
3775
3776void CGOpenMPRuntime::emitDistributeStaticInit(
3777 CodeGenFunction &CGF, SourceLocation Loc,
3778 OpenMPDistScheduleClauseKind SchedKind,
3779 const CGOpenMPRuntime::StaticRTInput &Values) {
3780 OpenMPSchedType ScheduleNum =
3781 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3782 llvm::Value *UpdatedLocation =
3783 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3784 llvm::Value *ThreadId = getThreadID(CGF, Loc);
3785 llvm::FunctionCallee StaticInitFunction =
3786 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3787 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3788 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3789 OMPC_SCHEDULE_MODIFIER_unknown, Values);
3790}
3791
3792void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3793 SourceLocation Loc,
3794 OpenMPDirectiveKind DKind) {
3795 if (!CGF.HaveInsertPoint())
3796 return;
3797 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3798 llvm::Value *Args[] = {
3799 emitUpdateLocation(CGF, Loc,
3800 isOpenMPDistributeDirective(DKind)
3801 ? OMP_IDENT_WORK_DISTRIBUTE
3802 : isOpenMPLoopDirective(DKind)
3803 ? OMP_IDENT_WORK_LOOP
3804 : OMP_IDENT_WORK_SECTIONS),
3805 getThreadID(CGF, Loc)};
3806 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3807 Args);
3808}
3809
3810void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3811 SourceLocation Loc,
3812 unsigned IVSize,
3813 bool IVSigned) {
3814 if (!CGF.HaveInsertPoint())
3815 return;
3816 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3817 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3818 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3819}
3820
3821llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3822 SourceLocation Loc, unsigned IVSize,
3823 bool IVSigned, Address IL,
3824 Address LB, Address UB,
3825 Address ST) {
3826 // Call __kmpc_dispatch_next(
3827 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3828 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3829 // kmp_int[32|64] *p_stride);
3830 llvm::Value *Args[] = {
3831 emitUpdateLocation(CGF, Loc),
3832 getThreadID(CGF, Loc),
3833 IL.getPointer(), // &isLastIter
3834 LB.getPointer(), // &Lower
3835 UB.getPointer(), // &Upper
3836 ST.getPointer() // &Stride
3837 };
3838 llvm::Value *Call =
3839 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3840 return CGF.EmitScalarConversion(
3841 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3842 CGF.getContext().BoolTy, Loc);
3843}
3844
3845void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3846 llvm::Value *NumThreads,
3847 SourceLocation Loc) {
3848 if (!CGF.HaveInsertPoint())
3849 return;
3850 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3851 llvm::Value *Args[] = {
3852 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3853 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3854 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3855 Args);
3856}
3857
3858void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3859 ProcBindKind ProcBind,
3860 SourceLocation Loc) {
3861 if (!CGF.HaveInsertPoint())
3862 return;
3863 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3863, __PRETTY_FUNCTION__))
;
3864 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3865 llvm::Value *Args[] = {
3866 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3867 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
3868 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3869}
3870
3871void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3872 SourceLocation Loc) {
3873 if (!CGF.HaveInsertPoint())
3874 return;
3875 // Build call void __kmpc_flush(ident_t *loc)
3876 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3877 emitUpdateLocation(CGF, Loc));
3878}
3879
3880namespace {
3881/// Indexes of fields for type kmp_task_t.
3882enum KmpTaskTFields {
3883 /// List of shared variables.
3884 KmpTaskTShareds,
3885 /// Task routine.
3886 KmpTaskTRoutine,
3887 /// Partition id for the untied tasks.
3888 KmpTaskTPartId,
3889 /// Function with call of destructors for private variables.
3890 Data1,
3891 /// Task priority.
3892 Data2,
3893 /// (Taskloops only) Lower bound.
3894 KmpTaskTLowerBound,
3895 /// (Taskloops only) Upper bound.
3896 KmpTaskTUpperBound,
3897 /// (Taskloops only) Stride.
3898 KmpTaskTStride,
3899 /// (Taskloops only) Is last iteration flag.
3900 KmpTaskTLastIter,
3901 /// (Taskloops only) Reduction data.
3902 KmpTaskTReductions,
3903};
3904} // anonymous namespace
3905
3906bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3907 return OffloadEntriesTargetRegion.empty() &&
3908 OffloadEntriesDeviceGlobalVar.empty();
3909}
3910
3911/// Initialize target region entry.
3912void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3913 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3914 StringRef ParentName, unsigned LineNum,
3915 unsigned Order) {
3916 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3918, __PRETTY_FUNCTION__))
3917 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3918, __PRETTY_FUNCTION__))
3918 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3918, __PRETTY_FUNCTION__))
;
3919 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3920 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3921 OMPTargetRegionEntryTargetRegion);
3922 ++OffloadingEntriesNum;
3923}
3924
3925void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3926 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3927 StringRef ParentName, unsigned LineNum,
3928 llvm::Constant *Addr, llvm::Constant *ID,
3929 OMPTargetRegionEntryKind Flags) {
3930 // If we are emitting code for a target, the entry is already initialized,
3931 // only has to be registered.
3932 if (CGM.getLangOpts().OpenMPIsDevice) {
3933 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3934 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3935 DiagnosticsEngine::Error,
3936 "Unable to find target region on line '%0' in the device code.");
3937 CGM.getDiags().Report(DiagID) << LineNum;
3938 return;
3939 }
3940 auto &Entry =
3941 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3942 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3942, __PRETTY_FUNCTION__))
;
3943 Entry.setAddress(Addr);
3944 Entry.setID(ID);
3945 Entry.setFlags(Flags);
3946 } else {
3947 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3948 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3949 ++OffloadingEntriesNum;
3950 }
3951}
3952
3953bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3954 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3955 unsigned LineNum) const {
3956 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3957 if (PerDevice == OffloadEntriesTargetRegion.end())
3958 return false;
3959 auto PerFile = PerDevice->second.find(FileID);
3960 if (PerFile == PerDevice->second.end())
3961 return false;
3962 auto PerParentName = PerFile->second.find(ParentName);
3963 if (PerParentName == PerFile->second.end())
3964 return false;
3965 auto PerLine = PerParentName->second.find(LineNum);
3966 if (PerLine == PerParentName->second.end())
3967 return false;
3968 // Fail if this entry is already registered.
3969 if (PerLine->second.getAddress() || PerLine->second.getID())
3970 return false;
3971 return true;
3972}
3973
3974void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3975 const OffloadTargetRegionEntryInfoActTy &Action) {
3976 // Scan all target region entries and perform the provided action.
3977 for (const auto &D : OffloadEntriesTargetRegion)
3978 for (const auto &F : D.second)
3979 for (const auto &P : F.second)
3980 for (const auto &L : P.second)
3981 Action(D.first, F.first, P.first(), L.first, L.second);
3982}
3983
3984void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3985 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3986 OMPTargetGlobalVarEntryKind Flags,
3987 unsigned Order) {
3988 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3990, __PRETTY_FUNCTION__))
3989 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3990, __PRETTY_FUNCTION__))
3990 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3990, __PRETTY_FUNCTION__))
;
3991 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3992 ++OffloadingEntriesNum;
3993}
3994
3995void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3996 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3997 CharUnits VarSize,
3998 OMPTargetGlobalVarEntryKind Flags,
3999 llvm::GlobalValue::LinkageTypes Linkage) {
4000 if (CGM.getLangOpts().OpenMPIsDevice) {
4001 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4002 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4003, __PRETTY_FUNCTION__))
4003 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4003, __PRETTY_FUNCTION__))
;
4004 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4005, __PRETTY_FUNCTION__))
4005 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4005, __PRETTY_FUNCTION__))
;
4006 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
4007 if (Entry.getVarSize().isZero()) {
4008 Entry.setVarSize(VarSize);
4009 Entry.setLinkage(Linkage);
4010 }
4011 return;
4012 }
4013 Entry.setVarSize(VarSize);
4014 Entry.setLinkage(Linkage);
4015 Entry.setAddress(Addr);
4016 } else {
4017 if (hasDeviceGlobalVarEntryInfo(VarName)) {
4018 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4019 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4020, __PRETTY_FUNCTION__))
4020 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4020, __PRETTY_FUNCTION__))
;
4021 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4022, __PRETTY_FUNCTION__))
4022 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4022, __PRETTY_FUNCTION__))
;
4023 if (Entry.getVarSize().isZero()) {
4024 Entry.setVarSize(VarSize);
4025 Entry.setLinkage(Linkage);
4026 }
4027 return;
4028 }
4029 OffloadEntriesDeviceGlobalVar.try_emplace(
4030 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4031 ++OffloadingEntriesNum;
4032 }
4033}
4034
4035void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4036 actOnDeviceGlobalVarEntriesInfo(
4037 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4038 // Scan all target region entries and perform the provided action.
4039 for (const auto &E : OffloadEntriesDeviceGlobalVar)
4040 Action(E.getKey(), E.getValue());
4041}
4042
4043void CGOpenMPRuntime::createOffloadEntry(
4044 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4045 llvm::GlobalValue::LinkageTypes Linkage) {
4046 StringRef Name = Addr->getName();
4047 llvm::Module &M = CGM.getModule();
4048 llvm::LLVMContext &C = M.getContext();
4049
4050 // Create constant string with the name.
4051 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4052
4053 std::string StringName = getName({"omp_offloading", "entry_name"});
4054 auto *Str = new llvm::GlobalVariable(
4055 M, StrPtrInit->getType(), /*isConstant=*/true,
4056 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4057 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4058
4059 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4060 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4061 llvm::ConstantInt::get(CGM.SizeTy, Size),
4062 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4063 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4064 std::string EntryName = getName({"omp_offloading", "entry", ""});
4065 llvm::GlobalVariable *Entry = createGlobalStruct(
4066 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4067 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4068
4069 // The entry has to be created in the section the linker expects it to be.
4070 Entry->setSection("omp_offloading_entries");
4071}
4072
4073void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4074 // Emit the offloading entries and metadata so that the device codegen side
4075 // can easily figure out what to emit. The produced metadata looks like
4076 // this:
4077 //
4078 // !omp_offload.info = !{!1, ...}
4079 //
4080 // Right now we only generate metadata for function that contain target
4081 // regions.
4082
4083 // If we are in simd mode or there are no entries, we don't need to do
4084 // anything.
4085 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
4086 return;
4087
4088 llvm::Module &M = CGM.getModule();
4089 llvm::LLVMContext &C = M.getContext();
4090 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4091 SourceLocation, StringRef>,
4092 16>
4093 OrderedEntries(OffloadEntriesInfoManager.size());
4094 llvm::SmallVector<StringRef, 16> ParentFunctions(
4095 OffloadEntriesInfoManager.size());
4096
4097 // Auxiliary methods to create metadata values and strings.
4098 auto &&GetMDInt = [this](unsigned V) {
4099 return llvm::ConstantAsMetadata::get(
4100 llvm::ConstantInt::get(CGM.Int32Ty, V));
4101 };
4102
4103 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4104
4105 // Create the offloading info metadata node.
4106 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4107
4108 // Create function that emits metadata for each target region entry;
4109 auto &&TargetRegionMetadataEmitter =
4110 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4111 &GetMDString](
4112 unsigned DeviceID, unsigned FileID, StringRef ParentName,
4113 unsigned Line,
4114 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4115 // Generate metadata for target regions. Each entry of this metadata
4116 // contains:
4117 // - Entry 0 -> Kind of this type of metadata (0).
4118 // - Entry 1 -> Device ID of the file where the entry was identified.
4119 // - Entry 2 -> File ID of the file where the entry was identified.
4120 // - Entry 3 -> Mangled name of the function where the entry was
4121 // identified.
4122 // - Entry 4 -> Line in the file where the entry was identified.
4123 // - Entry 5 -> Order the entry was created.
4124 // The first element of the metadata node is the kind.
4125 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4126 GetMDInt(FileID), GetMDString(ParentName),
4127 GetMDInt(Line), GetMDInt(E.getOrder())};
4128
4129 SourceLocation Loc;
4130 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4131 E = CGM.getContext().getSourceManager().fileinfo_end();
4132 I != E; ++I) {
4133 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4134 I->getFirst()->getUniqueID().getFile() == FileID) {
4135 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4136 I->getFirst(), Line, 1);
4137 break;
4138 }
4139 }
4140 // Save this entry in the right position of the ordered entries array.
4141 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4142 ParentFunctions[E.getOrder()] = ParentName;
4143
4144 // Add metadata to the named metadata node.
4145 MD->addOperand(llvm::MDNode::get(C, Ops));
4146 };
4147
4148 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4149 TargetRegionMetadataEmitter);
4150
4151 // Create function that emits metadata for each device global variable entry;
4152 auto &&DeviceGlobalVarMetadataEmitter =
4153 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4154 MD](StringRef MangledName,
4155 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4156 &E) {
4157 // Generate metadata for global variables. Each entry of this metadata
4158 // contains:
4159 // - Entry 0 -> Kind of this type of metadata (1).
4160 // - Entry 1 -> Mangled name of the variable.
4161 // - Entry 2 -> Declare target kind.
4162 // - Entry 3 -> Order the entry was created.
4163 // The first element of the metadata node is the kind.
4164 llvm::Metadata *Ops[] = {
4165 GetMDInt(E.getKind()), GetMDString(MangledName),
4166 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4167
4168 // Save this entry in the right position of the ordered entries array.
4169 OrderedEntries[E.getOrder()] =
4170 std::make_tuple(&E, SourceLocation(), MangledName);
4171
4172 // Add metadata to the named metadata node.
4173 MD->addOperand(llvm::MDNode::get(C, Ops));
4174 };
4175
4176 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4177 DeviceGlobalVarMetadataEmitter);
4178
4179 for (const auto &E : OrderedEntries) {
4180 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4180, __PRETTY_FUNCTION__))
;
4181 if (const auto *CE =
4182 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4183 std::get<0>(E))) {
4184 if (!CE->getID() || !CE->getAddress()) {
4185 // Do not blame the entry if the parent funtion is not emitted.
4186 StringRef FnName = ParentFunctions[CE->getOrder()];
4187 if (!CGM.GetGlobalValue(FnName))
4188 continue;
4189 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4190 DiagnosticsEngine::Error,
4191 "Offloading entry for target region in %0 is incorrect: either the "
4192 "address or the ID is invalid.");
4193 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4194 continue;
4195 }
4196 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4197 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4198 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
4199 OffloadEntryInfoDeviceGlobalVar>(
4200 std::get<0>(E))) {
4201 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4202 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4203 CE->getFlags());
4204 switch (Flags) {
4205 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4206 if (CGM.getLangOpts().OpenMPIsDevice &&
4207 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4208 continue;
4209 if (!CE->getAddress()) {
4210 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4211 DiagnosticsEngine::Error, "Offloading entry for declare target "
4212 "variable %0 is incorrect: the "
4213 "address is invalid.");
4214 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4215 continue;
4216 }
4217 // The vaiable has no definition - no need to add the entry.
4218 if (CE->getVarSize().isZero())
4219 continue;
4220 break;
4221 }
4222 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4223 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4225, __PRETTY_FUNCTION__))
4224 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4225, __PRETTY_FUNCTION__))
4225 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4225, __PRETTY_FUNCTION__))
;
4226 if (CGM.getLangOpts().OpenMPIsDevice)
4227 continue;
4228 if (!CE->getAddress()) {
4229 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4230 DiagnosticsEngine::Error,
4231 "Offloading entry for declare target variable is incorrect: the "
4232 "address is invalid.");
4233 CGM.getDiags().Report(DiagID);
4234 continue;
4235 }
4236 break;
4237 }
4238 createOffloadEntry(CE->getAddress(), CE->getAddress(),
4239 CE->getVarSize().getQuantity(), Flags,
4240 CE->getLinkage());
4241 } else {
4242 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4242)
;
4243 }
4244 }
4245}
4246
4247/// Loads all the offload entries information from the host IR
4248/// metadata.
4249void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4250 // If we are in target mode, load the metadata from the host IR. This code has
4251 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4252
4253 if (!CGM.getLangOpts().OpenMPIsDevice)
4254 return;
4255
4256 if (CGM.getLangOpts().OMPHostIRFile.empty())
4257 return;
4258
4259 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4260 if (auto EC = Buf.getError()) {
4261 CGM.getDiags().Report(diag::err_cannot_open_file)
4262 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4263 return;
4264 }
4265
4266 llvm::LLVMContext C;
4267 auto ME = expectedToErrorOrAndEmitErrors(
4268 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4269
4270 if (auto EC = ME.getError()) {
4271 unsigned DiagID = CGM.getDiags().getCustomDiagID(
4272 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4273 CGM.getDiags().Report(DiagID)
4274 << CGM.getLangOpts().OMPHostIRFile << EC.message();
4275 return;
4276 }
4277
4278 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4279 if (!MD)
4280 return;
4281
4282 for (llvm::MDNode *MN : MD->operands()) {
4283 auto &&GetMDInt = [MN](unsigned Idx) {
4284 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4285 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4286 };
4287
4288 auto &&GetMDString = [MN](unsigned Idx) {
4289 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4290 return V->getString();
4291 };
4292
4293 switch (GetMDInt(0)) {
4294 default:
4295 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4295)
;
4296 break;
4297 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4298 OffloadingEntryInfoTargetRegion:
4299 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4300 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4301 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4302 /*Order=*/GetMDInt(5));
4303 break;
4304 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4305 OffloadingEntryInfoDeviceGlobalVar:
4306 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4307 /*MangledName=*/GetMDString(1),
4308 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4309 /*Flags=*/GetMDInt(2)),
4310 /*Order=*/GetMDInt(3));
4311 break;
4312 }
4313 }
4314}
4315
4316void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4317 if (!KmpRoutineEntryPtrTy) {
4318 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4319 ASTContext &C = CGM.getContext();
4320 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4321 FunctionProtoType::ExtProtoInfo EPI;
4322 KmpRoutineEntryPtrQTy = C.getPointerType(
4323 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4324 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4325 }
4326}
4327
4328QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4329 // Make sure the type of the entry is already created. This is the type we
4330 // have to create:
4331 // struct __tgt_offload_entry{
4332 // void *addr; // Pointer to the offload entry info.
4333 // // (function or global)
4334 // char *name; // Name of the function or global.
4335 // size_t size; // Size of the entry info (0 if it a function).
4336 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
4337 // int32_t reserved; // Reserved, to use by the runtime library.
4338 // };
4339 if (TgtOffloadEntryQTy.isNull()) {
4340 ASTContext &C = CGM.getContext();
4341 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4342 RD->startDefinition();
4343 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4344 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4345 addFieldToRecordDecl(C, RD, C.getSizeType());
4346 addFieldToRecordDecl(
4347 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4348 addFieldToRecordDecl(
4349 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4350 RD->completeDefinition();
4351 RD->addAttr(PackedAttr::CreateImplicit(C));
4352 TgtOffloadEntryQTy = C.getRecordType(RD);
4353 }
4354 return TgtOffloadEntryQTy;
4355}
4356
4357namespace {
4358struct PrivateHelpersTy {
4359 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4360 const VarDecl *PrivateElemInit)
4361 : Original(Original), PrivateCopy(PrivateCopy),
4362 PrivateElemInit(PrivateElemInit) {}
4363 const VarDecl *Original;
4364 const VarDecl *PrivateCopy;
4365 const VarDecl *PrivateElemInit;
4366};
4367typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4368} // anonymous namespace
4369
4370static RecordDecl *
4371createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4372 if (!Privates.empty()) {
4373 ASTContext &C = CGM.getContext();
4374 // Build struct .kmp_privates_t. {
4375 // /* private vars */
4376 // };
4377 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4378 RD->startDefinition();
4379 for (const auto &Pair : Privates) {
4380 const VarDecl *VD = Pair.second.Original;
4381 QualType Type = VD->getType().getNonReferenceType();
4382 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4383 if (VD->hasAttrs()) {
4384 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4385 E(VD->getAttrs().end());
4386 I != E; ++I)
4387 FD->addAttr(*I);
4388 }
4389 }
4390 RD->completeDefinition();
4391 return RD;
4392 }
4393 return nullptr;
4394}
4395
4396static RecordDecl *
4397createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4398 QualType KmpInt32Ty,
4399 QualType KmpRoutineEntryPointerQTy) {
4400 ASTContext &C = CGM.getContext();
4401 // Build struct kmp_task_t {
4402 // void * shareds;
4403 // kmp_routine_entry_t routine;
4404 // kmp_int32 part_id;
4405 // kmp_cmplrdata_t data1;
4406 // kmp_cmplrdata_t data2;
4407 // For taskloops additional fields:
4408 // kmp_uint64 lb;
4409 // kmp_uint64 ub;
4410 // kmp_int64 st;
4411 // kmp_int32 liter;
4412 // void * reductions;
4413 // };
4414 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4415 UD->startDefinition();
4416 addFieldToRecordDecl(C, UD, KmpInt32Ty);
4417 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4418 UD->completeDefinition();
4419 QualType KmpCmplrdataTy = C.getRecordType(UD);
4420 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4421 RD->startDefinition();
4422 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4423 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4424 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4425 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4426 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4427 if (isOpenMPTaskLoopDirective(Kind)) {
4428 QualType KmpUInt64Ty =
4429 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4430 QualType KmpInt64Ty =
4431 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4432 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4433 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4434 addFieldToRecordDecl(C, RD, KmpInt64Ty);
4435 addFieldToRecordDecl(C, RD, KmpInt32Ty);
4436 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4437 }
4438 RD->completeDefinition();
4439 return RD;
4440}
4441
4442static RecordDecl *
4443createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4444 ArrayRef<PrivateDataTy> Privates) {
4445 ASTContext &C = CGM.getContext();
4446 // Build struct kmp_task_t_with_privates {
4447 // kmp_task_t task_data;
4448 // .kmp_privates_t. privates;
4449 // };
4450 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4451 RD->startDefinition();
4452 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4453 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4454 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4455 RD->completeDefinition();
4456 return RD;
4457}
4458
4459/// Emit a proxy function which accepts kmp_task_t as the second
4460/// argument.
4461/// \code
4462/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4463/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4464/// For taskloops:
4465/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4466/// tt->reductions, tt->shareds);
4467/// return 0;
4468/// }
4469/// \endcode
4470static llvm::Function *
4471emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4472 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4473 QualType KmpTaskTWithPrivatesPtrQTy,
4474 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4475 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4476 llvm::Value *TaskPrivatesMap) {
4477 ASTContext &C = CGM.getContext();
4478 FunctionArgList Args;
4479 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4480 ImplicitParamDecl::Other);
4481 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4482 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4483 ImplicitParamDecl::Other);
4484 Args.push_back(&GtidArg);
4485 Args.push_back(&TaskTypeArg);
4486 const auto &TaskEntryFnInfo =
4487 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4488 llvm::FunctionType *TaskEntryTy =
4489 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4490 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4491 auto *TaskEntry = llvm::Function::Create(
4492 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4493 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4494 TaskEntry->setDoesNotRecurse();
4495 CodeGenFunction CGF(CGM);
4496 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4497 Loc, Loc);
4498
4499 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4500 // tt,
4501 // For taskloops:
4502 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4503 // tt->task_data.shareds);
4504 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4505 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4506 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4507 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4508 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4509 const auto *KmpTaskTWithPrivatesQTyRD =
4510 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4511 LValue Base =
4512 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4513 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4514 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4515 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4516 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4517
4518 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4519 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4520 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4521 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4522 CGF.ConvertTypeForMem(SharedsPtrTy));
4523
4524 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4525 llvm::Value *PrivatesParam;
4526 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4527 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4528 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4529 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
4530 } else {
4531 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4532 }
4533
4534 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4535 TaskPrivatesMap,
4536 CGF.Builder
4537 .CreatePointerBitCastOrAddrSpaceCast(
4538 TDBase.getAddress(CGF), CGF.VoidPtrTy)
4539 .getPointer()};
4540 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4541 std::end(CommonArgs));
4542 if (isOpenMPTaskLoopDirective(Kind)) {
4543 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4544 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4545 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4546 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4547 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4548 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4549 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4550 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4551 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4552 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4553 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4554 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4555 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4556 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4557 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4558 CallArgs.push_back(LBParam);
4559 CallArgs.push_back(UBParam);
4560 CallArgs.push_back(StParam);
4561 CallArgs.push_back(LIParam);
4562 CallArgs.push_back(RParam);
4563 }
4564 CallArgs.push_back(SharedsParam);
4565
4566 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4567 CallArgs);
4568 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4569 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4570 CGF.FinishFunction();
4571 return TaskEntry;
4572}
4573
4574static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4575 SourceLocation Loc,
4576 QualType KmpInt32Ty,
4577 QualType KmpTaskTWithPrivatesPtrQTy,
4578 QualType KmpTaskTWithPrivatesQTy) {
4579 ASTContext &C = CGM.getContext();
4580 FunctionArgList Args;
4581 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4582 ImplicitParamDecl::Other);
4583 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4584 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4585 ImplicitParamDecl::Other);
4586 Args.push_back(&GtidArg);
4587 Args.push_back(&TaskTypeArg);
4588 const auto &DestructorFnInfo =
4589 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4590 llvm::FunctionType *DestructorFnTy =
4591 CGM.getTypes().GetFunctionType(DestructorFnInfo);
4592 std::string Name =
4593 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4594 auto *DestructorFn =
4595 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4596 Name, &CGM.getModule());
4597 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4598 DestructorFnInfo);
4599 DestructorFn->setDoesNotRecurse();
4600 CodeGenFunction CGF(CGM);
4601 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4602 Args, Loc, Loc);
4603
4604 LValue Base = CGF.EmitLoadOfPointerLValue(
4605 CGF.GetAddrOfLocalVar(&TaskTypeArg),
4606 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4607 const auto *KmpTaskTWithPrivatesQTyRD =
4608 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4609 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4610 Base = CGF.EmitLValueForField(Base, *FI);
4611 for (const auto *Field :
4612 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4613 if (QualType::DestructionKind DtorKind =
4614 Field->getType().isDestructedType()) {
4615 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4616 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
4617 }
4618 }
4619 CGF.FinishFunction();
4620 return DestructorFn;
4621}
4622
4623/// Emit a privates mapping function for correct handling of private and
4624/// firstprivate variables.
4625/// \code
4626/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4627/// **noalias priv1,..., <tyn> **noalias privn) {
4628/// *priv1 = &.privates.priv1;
4629/// ...;
4630/// *privn = &.privates.privn;
4631/// }
4632/// \endcode
4633static llvm::Value *
4634emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4635 ArrayRef<const Expr *> PrivateVars,
4636 ArrayRef<const Expr *> FirstprivateVars,
4637 ArrayRef<const Expr *> LastprivateVars,
4638 QualType PrivatesQTy,
4639 ArrayRef<PrivateDataTy> Privates) {
4640 ASTContext &C = CGM.getContext();
4641 FunctionArgList Args;
4642 ImplicitParamDecl TaskPrivatesArg(
4643 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4644 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4645 ImplicitParamDecl::Other);
4646 Args.push_back(&TaskPrivatesArg);
4647 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4648 unsigned Counter = 1;
4649 for (const Expr *E : PrivateVars) {
4650 Args.push_back(ImplicitParamDecl::Create(
4651 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4652 C.getPointerType(C.getPointerType(E->getType()))
4653 .withConst()
4654 .withRestrict(),
4655 ImplicitParamDecl::Other));
4656 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4657 PrivateVarsPos[VD] = Counter;
4658 ++Counter;
4659 }
4660 for (const Expr *E : FirstprivateVars) {
4661 Args.push_back(ImplicitParamDecl::Create(
4662 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4663 C.getPointerType(C.getPointerType(E->getType()))
4664 .withConst()
4665 .withRestrict(),
4666 ImplicitParamDecl::Other));
4667 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4668 PrivateVarsPos[VD] = Counter;
4669 ++Counter;
4670 }
4671 for (const Expr *E : LastprivateVars) {
4672 Args.push_back(ImplicitParamDecl::Create(
4673 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4674 C.getPointerType(C.getPointerType(E->getType()))
4675 .withConst()
4676 .withRestrict(),
4677 ImplicitParamDecl::Other));
4678 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4679 PrivateVarsPos[VD] = Counter;
4680 ++Counter;
4681 }
4682 const auto &TaskPrivatesMapFnInfo =
4683 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4684 llvm::FunctionType *TaskPrivatesMapTy =
4685 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4686 std::string Name =
4687 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4688 auto *TaskPrivatesMap = llvm::Function::Create(
4689 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4690 &CGM.getModule());
4691 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4692 TaskPrivatesMapFnInfo);
4693 if (CGM.getLangOpts().Optimize) {
4694 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4695 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4696 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4697 }
4698 CodeGenFunction CGF(CGM);
4699 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4700 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4701
4702 // *privi = &.privates.privi;
4703 LValue Base = CGF.EmitLoadOfPointerLValue(
4704 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4705 TaskPrivatesArg.getType()->castAs<PointerType>());
4706 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4707 Counter = 0;
4708 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4709 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4710 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4711 LValue RefLVal =
4712 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4713 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4714 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
4715 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
4716 ++Counter;
4717 }
4718 CGF.FinishFunction();
4719 return TaskPrivatesMap;
4720}
4721
4722/// Emit initialization for private variables in task-based directives.
4723static void emitPrivatesInit(CodeGenFunction &CGF,
4724 const OMPExecutableDirective &D,
4725 Address KmpTaskSharedsPtr, LValue TDBase,
4726 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4727 QualType SharedsTy, QualType SharedsPtrTy,
4728 const OMPTaskDataTy &Data,
4729 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4730 ASTContext &C = CGF.getContext();
4731 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4732 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4733 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4734 ? OMPD_taskloop
4735 : OMPD_task;
4736 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4737 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4738 LValue SrcBase;
4739 bool IsTargetTask =
4740 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4741 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4742 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4743 // PointersArray and SizesArray. The original variables for these arrays are
4744 // not captured and we get their addresses explicitly.
4745 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4746 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4747 SrcBase = CGF.MakeAddrLValue(
4748 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4749 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4750 SharedsTy);
4751 }
4752 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4753 for (const PrivateDataTy &Pair : Privates) {
4754 const VarDecl *VD = Pair.second.PrivateCopy;
4755 const Expr *Init = VD->getAnyInitializer();
4756 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4757 !CGF.isTrivialInitializer(Init)))) {
4758 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4759 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4760 const VarDecl *OriginalVD = Pair.second.Original;
4761 // Check if the variable is the target-based BasePointersArray,
4762 // PointersArray or SizesArray.
4763 LValue SharedRefLValue;
4764 QualType Type = PrivateLValue.getType();
4765 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4766 if (IsTargetTask && !SharedField) {
4767 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4768 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4769 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4770 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4771 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4772 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4773 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
4774 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4774, __PRETTY_FUNCTION__))
;
4775 SharedRefLValue =
4776 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4777 } else {
4778 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4779 SharedRefLValue = CGF.MakeAddrLValue(
4780 Address(SharedRefLValue.getPointer(CGF),
4781 C.getDeclAlign(OriginalVD)),
4782 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4783 SharedRefLValue.getTBAAInfo());
4784 }
4785 if (Type->isArrayType()) {
4786 // Initialize firstprivate array.
4787 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4788 // Perform simple memcpy.
4789 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4790 } else {
4791 // Initialize firstprivate array using element-by-element
4792 // initialization.
4793 CGF.EmitOMPAggregateAssign(
4794 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
4795 Type,
4796 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4797 Address SrcElement) {
4798 // Clean up any temporaries needed by the initialization.
4799 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4800 InitScope.addPrivate(
4801 Elem, [SrcElement]() -> Address { return SrcElement; });
4802 (void)InitScope.Privatize();
4803 // Emit initialization for single element.
4804 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4805 CGF, &CapturesInfo);
4806 CGF.EmitAnyExprToMem(Init, DestElement,
4807 Init->getType().getQualifiers(),
4808 /*IsInitializer=*/false);
4809 });
4810 }
4811 } else {
4812 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4813 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
4814 return SharedRefLValue.getAddress(CGF);
4815 });
4816 (void)InitScope.Privatize();
4817 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4818 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4819 /*capturedByInit=*/false);
4820 }
4821 } else {
4822 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4823 }
4824 }
4825 ++FI;
4826 }
4827}
4828
4829/// Check if duplication function is required for taskloops.
4830static bool checkInitIsRequired(CodeGenFunction &CGF,
4831 ArrayRef<PrivateDataTy> Privates) {
4832 bool InitRequired = false;
4833 for (const PrivateDataTy &Pair : Privates) {
4834 const VarDecl *VD = Pair.second.PrivateCopy;
4835 const Expr *Init = VD->getAnyInitializer();
4836 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4837 !CGF.isTrivialInitializer(Init));
4838 if (InitRequired)
4839 break;
4840 }
4841 return InitRequired;
4842}
4843
4844
4845/// Emit task_dup function (for initialization of
4846/// private/firstprivate/lastprivate vars and last_iter flag)
4847/// \code
4848/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4849/// lastpriv) {
4850/// // setup lastprivate flag
4851/// task_dst->last = lastpriv;
4852/// // could be constructor calls here...
4853/// }
4854/// \endcode
4855static llvm::Value *
4856emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4857 const OMPExecutableDirective &D,
4858 QualType KmpTaskTWithPrivatesPtrQTy,
4859 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4860 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4861 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4862 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4863 ASTContext &C = CGM.getContext();
4864 FunctionArgList Args;
4865 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4866 KmpTaskTWithPrivatesPtrQTy,
4867 ImplicitParamDecl::Other);
4868 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4869 KmpTaskTWithPrivatesPtrQTy,
4870 ImplicitParamDecl::Other);
4871 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4872 ImplicitParamDecl::Other);
4873 Args.push_back(&DstArg);
4874 Args.push_back(&SrcArg);
4875 Args.push_back(&LastprivArg);
4876 const auto &TaskDupFnInfo =
4877 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4878 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4879 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4880 auto *TaskDup = llvm::Function::Create(
4881 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4882 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4883 TaskDup->setDoesNotRecurse();
4884 CodeGenFunction CGF(CGM);
4885 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4886 Loc);
4887
4888 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4889 CGF.GetAddrOfLocalVar(&DstArg),
4890 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4891 // task_dst->liter = lastpriv;
4892 if (WithLastIter) {
4893 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4894 LValue Base = CGF.EmitLValueForField(
4895 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4896 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4897 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4898 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4899 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4900 }
4901
4902 // Emit initial values for private copies (if any).
4903 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4903, __PRETTY_FUNCTION__))
;
4904 Address KmpTaskSharedsPtr = Address::invalid();
4905 if (!Data.FirstprivateVars.empty()) {
4906 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4907 CGF.GetAddrOfLocalVar(&SrcArg),
4908 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4909 LValue Base = CGF.EmitLValueForField(
4910 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4911 KmpTaskSharedsPtr = Address(
4912 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4913 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4914 KmpTaskTShareds)),
4915 Loc),
4916 CGF.getNaturalTypeAlignment(SharedsTy));
4917 }
4918 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4919 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4920 CGF.FinishFunction();
4921 return TaskDup;
4922}
4923
4924/// Checks if destructor function is required to be generated.
4925/// \return true if cleanups are required, false otherwise.
4926static bool
4927checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4928 bool NeedsCleanup = false;
4929 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4930 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4931 for (const FieldDecl *FD : PrivateRD->fields()) {
4932 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4933 if (NeedsCleanup)
4934 break;
4935 }
4936 return NeedsCleanup;
4937}
4938
4939CGOpenMPRuntime::TaskResultTy
4940CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4941 const OMPExecutableDirective &D,
4942 llvm::Function *TaskFunction, QualType SharedsTy,
4943 Address Shareds, const OMPTaskDataTy &Data) {
4944 ASTContext &C = CGM.getContext();
4945 llvm::SmallVector<PrivateDataTy, 4> Privates;
4946 // Aggregate privates and sort them by the alignment.
4947 auto I = Data.PrivateCopies.begin();
4948 for (const Expr *E : Data.PrivateVars) {
4949 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4950 Privates.emplace_back(
4951 C.getDeclAlign(VD),
4952 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4953 /*PrivateElemInit=*/nullptr));
4954 ++I;
4955 }
4956 I = Data.FirstprivateCopies.begin();
4957 auto IElemInitRef = Data.FirstprivateInits.begin();
4958 for (const Expr *E : Data.FirstprivateVars) {
4959 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4960 Privates.emplace_back(
4961 C.getDeclAlign(VD),
4962 PrivateHelpersTy(
4963 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4964 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4965 ++I;
4966 ++IElemInitRef;
4967 }
4968 I = Data.LastprivateCopies.begin();
4969 for (const Expr *E : Data.LastprivateVars) {
4970 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4971 Privates.emplace_back(
4972 C.getDeclAlign(VD),
4973 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4974 /*PrivateElemInit=*/nullptr));
4975 ++I;
4976 }
4977 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
4978 return L.first > R.first;
4979 });
4980 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4981 // Build type kmp_routine_entry_t (if not built yet).
4982 emitKmpRoutineEntryT(KmpInt32Ty);
4983 // Build type kmp_task_t (if not built yet).
4984 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4985 if (SavedKmpTaskloopTQTy.isNull()) {
4986 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4987 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4988 }
4989 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4990 } else {
4991 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4994, __PRETTY_FUNCTION__))
4992 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4994, __PRETTY_FUNCTION__))
4993 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4994, __PRETTY_FUNCTION__))
4994 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4994, __PRETTY_FUNCTION__))
;
4995 if (SavedKmpTaskTQTy.isNull()) {
4996 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4997 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4998 }
4999 KmpTaskTQTy = SavedKmpTaskTQTy;
5000 }
5001 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5002 // Build particular struct kmp_task_t for the given task.
5003 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5004 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5005 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5006 QualType KmpTaskTWithPrivatesPtrQTy =
5007 C.getPointerType(KmpTaskTWithPrivatesQTy);
5008 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5009 llvm::Type *KmpTaskTWithPrivatesPtrTy =
5010 KmpTaskTWithPrivatesTy->getPointerTo();
5011 llvm::Value *KmpTaskTWithPrivatesTySize =
5012 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5013 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5014
5015 // Emit initial values for private copies (if any).
5016 llvm::Value *TaskPrivatesMap = nullptr;
5017 llvm::Type *TaskPrivatesMapTy =
5018 std::next(TaskFunction->arg_begin(), 3)->getType();
5019 if (!Privates.empty()) {
5020 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5021 TaskPrivatesMap = emitTaskPrivateMappingFunction(
5022 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5023 FI->getType(), Privates);
5024 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5025 TaskPrivatesMap, TaskPrivatesMapTy);
5026 } else {
5027 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5028 cast<llvm::PointerType>(TaskPrivatesMapTy));
5029 }
5030 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5031 // kmp_task_t *tt);
5032 llvm::Function *TaskEntry = emitProxyTaskFunction(
5033 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5034 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5035 TaskPrivatesMap);
5036
5037 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5038 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5039 // kmp_routine_entry_t *task_entry);
5040 // Task flags. Format is taken from
5041 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5042 // description of kmp_tasking_flags struct.
5043 enum {
5044 TiedFlag = 0x1,
5045 FinalFlag = 0x2,
5046 DestructorsFlag = 0x8,
5047 PriorityFlag = 0x20
5048 };
5049 unsigned Flags = Data.Tied ? TiedFlag : 0;
5050 bool NeedsCleanup = false;
5051 if (!Privates.empty()) {
5052 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5053 if (NeedsCleanup)
5054 Flags = Flags | DestructorsFlag;
5055 }
5056 if (Data.Priority.getInt())
5057 Flags = Flags | PriorityFlag;
5058 llvm::Value *TaskFlags =
5059 Data.Final.getPointer()
5060 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5061 CGF.Builder.getInt32(FinalFlag),
5062 CGF.Builder.getInt32(/*C=*/0))
5063 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5064 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5065 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5066 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5067 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5068 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5069 TaskEntry, KmpRoutineEntryPtrTy)};
5070 llvm::Value *NewTask;
5071 if (D.hasClausesOfKind<OMPNowaitClause>()) {
5072 // Check if we have any device clause associated with the directive.
5073 const Expr *Device = nullptr;
5074 if (auto *C = D.getSingleClause<OMPDeviceClause>())
5075 Device = C->getDevice();
5076 // Emit device ID if any otherwise use default value.
5077 llvm::Value *DeviceID;
5078 if (Device)
5079 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5080 CGF.Int64Ty, /*isSigned=*/true);
5081 else
5082 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5083 AllocArgs.push_back(DeviceID);
5084 NewTask = CGF.EmitRuntimeCall(
5085 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5086 } else {
5087 NewTask = CGF.EmitRuntimeCall(
5088 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5089 }
5090 llvm::Value *NewTaskNewTaskTTy =
5091 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5092 NewTask, KmpTaskTWithPrivatesPtrTy);
5093 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5094 KmpTaskTWithPrivatesQTy);
5095 LValue TDBase =
5096 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5097 // Fill the data in the resulting kmp_task_t record.
5098 // Copy shareds if there are any.
5099 Address KmpTaskSharedsPtr = Address::invalid();
5100 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5101 KmpTaskSharedsPtr =
5102 Address(CGF.EmitLoadOfScalar(
5103 CGF.EmitLValueForField(
5104 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5105 KmpTaskTShareds)),
5106 Loc),
5107 CGF.getNaturalTypeAlignment(SharedsTy));
5108 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5109 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5110 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5111 }
5112 // Emit initial values for private copies (if any).
5113 TaskResultTy Result;
5114 if (!Privates.empty()) {
5115 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5116 SharedsTy, SharedsPtrTy, Data, Privates,
5117 /*ForDup=*/false);
5118 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5119 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5120 Result.TaskDupFn = emitTaskDupFunction(
5121 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5122 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5123 /*WithLastIter=*/!Data.LastprivateVars.empty());
5124 }
5125 }
5126 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5127 enum { Priority = 0, Destructors = 1 };
5128 // Provide pointer to function with destructors for privates.
5129 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5130 const RecordDecl *KmpCmplrdataUD =
5131 (*FI)->getType()->getAsUnionType()->getDecl();
5132 if (NeedsCleanup) {
5133 llvm::Value *DestructorFn = emitDestructorsFunction(
5134 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5135 KmpTaskTWithPrivatesQTy);
5136 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5137 LValue DestructorsLV = CGF.EmitLValueForField(
5138 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5139 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5140 DestructorFn, KmpRoutineEntryPtrTy),
5141 DestructorsLV);
5142 }
5143 // Set priority.
5144 if (Data.Priority.getInt()) {
5145 LValue Data2LV = CGF.EmitLValueForField(
5146 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5147 LValue PriorityLV = CGF.EmitLValueForField(
5148 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5149 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5150 }
5151 Result.NewTask = NewTask;
5152 Result.TaskEntry = TaskEntry;
5153 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5154 Result.TDBase = TDBase;
5155 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5156 return Result;
5157}
5158
5159void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5160 const OMPExecutableDirective &D,
5161 llvm::Function *TaskFunction,
5162 QualType SharedsTy, Address Shareds,
5163 const Expr *IfCond,
5164 const OMPTaskDataTy &Data) {
5165 if (!CGF.HaveInsertPoint())
5166 return;
5167
5168 TaskResultTy Result =
5169 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5170 llvm::Value *NewTask = Result.NewTask;
5171 llvm::Function *TaskEntry = Result.TaskEntry;
5172 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5173 LValue TDBase = Result.TDBase;
5174 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5175 ASTContext &C = CGM.getContext();
5176 // Process list of dependences.
5177 Address DependenciesArray = Address::invalid();
5178 unsigned NumDependencies = Data.Dependences.size();
5179 if (NumDependencies) {
5180 // Dependence kind for RTL.
5181 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5182 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5183 RecordDecl *KmpDependInfoRD;
5184 QualType FlagsTy =
5185 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5186 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5187 if (KmpDependInfoTy.isNull()) {
5188 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5189 KmpDependInfoRD->startDefinition();
5190 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5191 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5192 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5193 KmpDependInfoRD->completeDefinition();
5194 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5195 } else {
5196 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5197 }
5198 // Define type kmp_depend_info[<Dependences.size()>];
5199 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5200 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5201 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
5202 // kmp_depend_info[<Dependences.size()>] deps;
5203 DependenciesArray =
5204 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5205 for (unsigned I = 0; I < NumDependencies; ++I) {
5206 const Expr *E = Data.Dependences[I].second;
5207 LValue Addr = CGF.EmitLValue(E);
5208 llvm::Value *Size;
5209 QualType Ty = E->getType();
5210 if (const auto *ASE =
5211 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5212 LValue UpAddrLVal =
5213 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5214 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
5215 UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
5216 llvm::Value *LowIntPtr =
5217 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy);
5218 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5219 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5220 } else {
5221 Size = CGF.getTypeSize(Ty);
5222 }
5223 LValue Base = CGF.MakeAddrLValue(
5224 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5225 KmpDependInfoTy);
5226 // deps[i].base_addr = &<Dependences[i].second>;
5227 LValue BaseAddrLVal = CGF.EmitLValueForField(
5228 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5229 CGF.EmitStoreOfScalar(
5230 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy),
5231 BaseAddrLVal);
5232 // deps[i].len = sizeof(<Dependences[i].second>);
5233 LValue LenLVal = CGF.EmitLValueForField(
5234 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5235 CGF.EmitStoreOfScalar(Size, LenLVal);
5236 // deps[i].flags = <Dependences[i].first>;
5237 RTLDependenceKindTy DepKind;
5238 switch (Data.Dependences[I].first) {
5239 case OMPC_DEPEND_in:
5240 DepKind = DepIn;
5241 break;
5242 // Out and InOut dependencies must use the same code.
5243 case OMPC_DEPEND_out:
5244 case OMPC_DEPEND_inout:
5245 DepKind = DepInOut;
5246 break;
5247 case OMPC_DEPEND_mutexinoutset:
5248 DepKind = DepMutexInOutSet;
5249 break;
5250 case OMPC_DEPEND_source:
5251 case OMPC_DEPEND_sink:
5252 case OMPC_DEPEND_unknown:
5253 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 5253)
;
5254 }
5255 LValue FlagsLVal = CGF.EmitLValueForField(
5256 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5257 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5258 FlagsLVal);
5259 }
5260 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5261 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5262 }
5263
5264 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5265 // libcall.
5266 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5267 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5268 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5269 // list is not empty
5270 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5271 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5272 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5273 llvm::Value *DepTaskArgs[7];
5274 if (NumDependencies) {
5275 DepTaskArgs[0] = UpLoc;
5276 DepTaskArgs[1] = ThreadID;
5277 DepTaskArgs[2] = NewTask;
5278 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5279 DepTaskArgs[4] = DependenciesArray.getPointer();
5280 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5281 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5282 }
5283 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5284 &TaskArgs,
5285 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5286 if (!Data.Tied) {
5287 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5288 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5289 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5290 }
5291 if (NumDependencies) {
5292 CGF.EmitRuntimeCall(
5293 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5294 } else {
5295 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5296 TaskArgs);
5297 }
5298 // Check if parent region is untied and build return for untied task;
5299 if (auto *Region =
5300 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5301 Region->emitUntiedSwitch(CGF);
5302 };
5303
5304 llvm::Value *DepWaitTaskArgs[6];
5305 if (NumDependencies) {
5306 DepWaitTaskArgs[0] = UpLoc;
5307 DepWaitTaskArgs[1] = ThreadID;
5308 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5309 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5310 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5311 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5312 }
5313 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5314 NumDependencies, &DepWaitTaskArgs,
5315 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5316 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5317 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5318 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5319 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5320 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5321 // is specified.
5322 if (NumDependencies)
5323 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5324 DepWaitTaskArgs);
5325 // Call proxy_task_entry(gtid, new_task);
5326 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5327 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5328 Action.Enter(CGF);
5329 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5330 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5331 OutlinedFnArgs);
5332 };
5333
5334 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5335 // kmp_task_t *new_task);
5336 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5337 // kmp_task_t *new_task);
5338 RegionCodeGenTy RCG(CodeGen);
5339 CommonActionTy Action(
5340 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5341 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5342 RCG.setAction(Action);
5343 RCG(CGF);
5344 };
5345
5346 if (IfCond) {
5347 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5348 } else {
5349 RegionCodeGenTy ThenRCG(ThenCodeGen);
5350 ThenRCG(CGF);
5351 }
5352}
5353
5354void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5355 const OMPLoopDirective &D,
5356 llvm::Function *TaskFunction,
5357 QualType SharedsTy, Address Shareds,
5358 const Expr *IfCond,
5359 const OMPTaskDataTy &Data) {
5360 if (!CGF.HaveInsertPoint())
5361 return;
5362 TaskResultTy Result =
5363 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5364 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5365 // libcall.
5366 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5367 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5368 // sched, kmp_uint64 grainsize, void *task_dup);
5369 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5370 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5371 llvm::Value *IfVal;
5372 if (IfCond) {
5373 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5374 /*isSigned=*/true);
5375 } else {
5376 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5377 }
5378
5379 LValue LBLVal = CGF.EmitLValueForField(
5380 Result.TDBase,
5381 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5382 const auto *LBVar =
5383 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5384 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5385 LBLVal.getQuals(),
5386 /*IsInitializer=*/true);
5387 LValue UBLVal = CGF.EmitLValueForField(
5388 Result.TDBase,
5389 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5390 const auto *UBVar =
5391 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5392 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5393 UBLVal.getQuals(),
5394 /*IsInitializer=*/true);
5395 LValue StLVal = CGF.EmitLValueForField(
5396 Result.TDBase,
5397 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5398 const auto *StVar =
5399 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5400 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5401 StLVal.getQuals(),
5402 /*IsInitializer=*/true);
5403 // Store reductions address.
5404 LValue RedLVal = CGF.EmitLValueForField(
5405 Result.TDBase,
5406 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5407 if (Data.Reductions) {
5408 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5409 } else {
5410 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5411 CGF.getContext().VoidPtrTy);
5412 }
5413 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5414 llvm::Value *TaskArgs[] = {
5415 UpLoc,
5416 ThreadID,
5417 Result.NewTask,
5418 IfVal,
5419 LBLVal.getPointer(CGF),
5420 UBLVal.getPointer(CGF),
5421 CGF.EmitLoadOfScalar(StLVal, Loc),
5422 llvm::ConstantInt::getSigned(
5423 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5424 llvm::ConstantInt::getSigned(
5425 CGF.IntTy, Data.Schedule.getPointer()
5426 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5427 : NoSchedule),
5428 Data.Schedule.getPointer()
5429 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5430 /*isSigned=*/false)
5431 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5432 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5433 Result.TaskDupFn, CGF.VoidPtrTy)
5434 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5435 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5436}
5437
5438/// Emit reduction operation for each element of array (required for
5439/// array sections) LHS op = RHS.
5440/// \param Type Type of array.
5441/// \param LHSVar Variable on the left side of the reduction operation
5442/// (references element of array in original variable).
5443/// \param RHSVar Variable on the right side of the reduction operation
5444/// (references element of array in original variable).
5445/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5446/// RHSVar.
5447static void EmitOMPAggregateReduction(
5448 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5449 const VarDecl *RHSVar,
5450 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5451 const Expr *, const Expr *)> &RedOpGen,
5452 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5453 const Expr *UpExpr = nullptr) {
5454 // Perform element-by-element initialization.
5455 QualType ElementTy;
5456 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5457 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5458
5459 // Drill down to the base element type on both arrays.
5460 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5461 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5462
5463 llvm::Value *RHSBegin = RHSAddr.getPointer();
5464 llvm::Value *LHSBegin = LHSAddr.getPointer();
5465 // Cast from pointer to array type to pointer to single element.
5466 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5467 // The basic structure here is a while-do loop.
5468 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5469 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5470 llvm::Value *IsEmpty =
5471 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5472 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5473
5474 // Enter the loop body, making that address the current address.
5475 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5476 CGF.EmitBlock(BodyBB);
5477
5478 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5479
5480 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5481 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5482 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5483 Address RHSElementCurrent =
5484 Address(RHSElementPHI,
5485 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5486
5487 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5488 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5489 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5490 Address LHSElementCurrent =
5491 Address(LHSElementPHI,
5492 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5493
5494 // Emit copy.
5495 CodeGenFunction::OMPPrivateScope Scope(CGF);
5496 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5497 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5498 Scope.Privatize();
5499 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5500 Scope.ForceCleanup();
5501
5502 // Shift the address forward by one element.
5503 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5504 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5505 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5506 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5507 // Check whether we've reached the end.
5508 llvm::Value *Done =
5509 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5510 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5511 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5512 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5513
5514 // Done.
5515 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5516}
5517
5518/// Emit reduction combiner. If the combiner is a simple expression emit it as
5519/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5520/// UDR combiner function.
5521static void emitReductionCombiner(CodeGenFunction &CGF,
5522 const Expr *ReductionOp) {
5523 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5524 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5525 if (const auto *DRE =
5526 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5527 if (const auto *DRD =
5528 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5529 std::pair<llvm::Function *, llvm::Function *> Reduction =
5530 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5531 RValue Func = RValue::get(Reduction.first);
5532 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5533 CGF.EmitIgnoredExpr(ReductionOp);
5534 return;
5535 }
5536 CGF.EmitIgnoredExpr(ReductionOp);
5537}
5538
5539llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5540 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5541 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5542 ArrayRef<const Expr *> ReductionOps) {
5543 ASTContext &C = CGM.getContext();
5544
5545 // void reduction_func(void *LHSArg, void *RHSArg);
5546 FunctionArgList Args;
5547 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5548 ImplicitParamDecl::Other);
5549 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5550 ImplicitParamDecl::Other);
5551 Args.push_back(&LHSArg);
5552 Args.push_back(&RHSArg);
5553 const auto &CGFI =
5554 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5555 std::string Name = getName({"omp", "reduction", "reduction_func"});
5556 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5557 llvm::GlobalValue::InternalLinkage, Name,
5558 &CGM.getModule());
5559 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5560 Fn->setDoesNotRecurse();
5561 CodeGenFunction CGF(CGM);
5562 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5563
5564 // Dst = (void*[n])(LHSArg);
5565 // Src = (void*[n])(RHSArg);
5566 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5567 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5568 ArgsType), CGF.getPointerAlign());
5569 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5570 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5571 ArgsType), CGF.getPointerAlign());
5572
5573 // ...
5574 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5575 // ...
5576 CodeGenFunction::OMPPrivateScope Scope(CGF);
5577 auto IPriv = Privates.begin();
5578 unsigned Idx = 0;
5579 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5580 const auto *RHSVar =
5581 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5582 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5583 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5584 });
5585 const auto *LHSVar =
5586 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5587 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5588 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5589 });
5590 QualType PrivTy = (*IPriv)->getType();
5591 if (PrivTy->isVariablyModifiedType()) {
5592 // Get array size and emit VLA type.
5593 ++Idx;
5594 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5595 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5596 const VariableArrayType *VLA =
5597 CGF.getContext().getAsVariableArrayType(PrivTy);
5598 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5599 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5600 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5601 CGF.EmitVariablyModifiedType(PrivTy);
5602 }
5603 }
5604 Scope.Privatize();
5605 IPriv = Privates.begin();
5606 auto ILHS = LHSExprs.begin();
5607 auto IRHS = RHSExprs.begin();
5608 for (const Expr *E : ReductionOps) {
5609 if ((*IPriv)->getType()->isArrayType()) {
5610 // Emit reduction for array section.
5611 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5612 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5613 EmitOMPAggregateReduction(
5614 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5615 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5616 emitReductionCombiner(CGF, E);
5617 });
5618 } else {
5619 // Emit reduction for array subscript or single variable.
5620 emitReductionCombiner(CGF, E);
5621 }
5622 ++IPriv;
5623 ++ILHS;
5624 ++IRHS;
5625 }
5626 Scope.ForceCleanup();
5627 CGF.FinishFunction();
5628 return Fn;
5629}
5630
5631void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5632 const Expr *ReductionOp,
5633 const Expr *PrivateRef,
5634 const DeclRefExpr *LHS,
5635 const DeclRefExpr *RHS) {
5636 if (PrivateRef->getType()->isArrayType()) {
5637 // Emit reduction for array section.
5638 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5639 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5640 EmitOMPAggregateReduction(
5641 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5642 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5643 emitReductionCombiner(CGF, ReductionOp);
5644 });
5645 } else {
5646 // Emit reduction for array subscript or single variable.
5647 emitReductionCombiner(CGF, ReductionOp);
5648 }
5649}
5650
5651void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5652 ArrayRef<const Expr *> Privates,
5653 ArrayRef<const Expr *> LHSExprs,
5654 ArrayRef<const Expr *> RHSExprs,
5655 ArrayRef<const Expr *> ReductionOps,
5656 ReductionOptionsTy Options) {
5657 if (!CGF.HaveInsertPoint())
5658 return;
5659
5660 bool WithNowait = Options.WithNowait;
5661 bool SimpleReduction = Options.SimpleReduction;
5662
5663 // Next code should be emitted for reduction:
5664 //
5665 // static kmp_critical_name lock = { 0 };
5666 //
5667 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5668 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5669 // ...
5670 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5671 // *(Type<n>-1*)rhs[<n>-1]);
5672 // }
5673 //
5674 // ...
5675 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5676 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5677 // RedList, reduce_func, &<lock>)) {
5678 // case 1:
5679 // ...
5680 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5681 // ...
5682 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5683 // break;
5684 // case 2:
5685 // ...
5686 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5687 // ...
5688 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5689 // break;
5690 // default:;
5691 // }
5692 //
5693 // if SimpleReduction is true, only the next code is generated:
5694 // ...
5695 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5696 // ...
5697
5698 ASTContext &C = CGM.getContext();
5699
5700 if (SimpleReduction) {
5701 CodeGenFunction::RunCleanupsScope Scope(CGF);
5702 auto IPriv = Privates.begin();
5703 auto ILHS = LHSExprs.begin();
5704 auto IRHS = RHSExprs.begin();
5705 for (const Expr *E : ReductionOps) {
5706 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5707 cast<DeclRefExpr>(*IRHS));
5708 ++IPriv;
5709 ++ILHS;
5710 ++IRHS;
5711 }
5712 return;
5713 }
5714
5715 // 1. Build a list of reduction variables.
5716 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5717 auto Size = RHSExprs.size();
5718 for (const Expr *E : Privates) {
5719 if (E->getType()->isVariablyModifiedType())
5720 // Reserve place for array size.
5721 ++Size;
5722 }
5723 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5724 QualType ReductionArrayTy =
5725 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5726 /*IndexTypeQuals=*/0);
5727 Address ReductionList =
5728