Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8541, column 15
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/include -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-15-222444-33637-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/FileManager.h"
25#include "clang/Basic/OpenMPKinds.h"
26#include "clang/Basic/SourceManager.h"
27#include "clang/CodeGen/ConstantInitBuilder.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/SetOperations.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/Bitcode/BitcodeReader.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Value.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40#include <numeric>
41
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46namespace {
47/// Base class for handling code generation inside OpenMP regions.
48class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
49public:
50 /// Kinds of OpenMP regions used in codegen.
51 enum CGOpenMPRegionKind {
52 /// Region with outlined function for standalone 'parallel'
53 /// directive.
54 ParallelOutlinedRegion,
55 /// Region with outlined function for standalone 'task' directive.
56 TaskOutlinedRegion,
57 /// Region for constructs that do not require function outlining,
58 /// like 'for', 'sections', 'atomic' etc. directives.
59 InlinedRegion,
60 /// Region with outlined function for standalone 'target' directive.
61 TargetRegion,
62 };
63
64 CGOpenMPRegionInfo(const CapturedStmt &CS,
65 const CGOpenMPRegionKind RegionKind,
66 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
67 bool HasCancel)
68 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
69 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
70
71 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
72 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
73 bool HasCancel)
74 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
75 Kind(Kind), HasCancel(HasCancel) {}
76
77 /// Get a variable or parameter for storing global thread id
78 /// inside OpenMP construct.
79 virtual const VarDecl *getThreadIDVariable() const = 0;
80
81 /// Emit the captured statement body.
82 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
83
84 /// Get an LValue for the current ThreadID variable.
85 /// \return LValue for thread id variable. This LValue always has type int32*.
86 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
87
88 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
89
90 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
91
92 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
93
94 bool hasCancel() const { return HasCancel; }
95
96 static bool classof(const CGCapturedStmtInfo *Info) {
97 return Info->getKind() == CR_OpenMP;
98 }
99
100 ~CGOpenMPRegionInfo() override = default;
101
102protected:
103 CGOpenMPRegionKind RegionKind;
104 RegionCodeGenTy CodeGen;
105 OpenMPDirectiveKind Kind;
106 bool HasCancel;
107};
108
109/// API for captured statement code generation in OpenMP constructs.
110class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
111public:
112 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
113 const RegionCodeGenTy &CodeGen,
114 OpenMPDirectiveKind Kind, bool HasCancel,
115 StringRef HelperName)
116 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
117 HasCancel),
118 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
119 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 119, __PRETTY_FUNCTION__))
;
120 }
121
122 /// Get a variable or parameter for storing global thread id
123 /// inside OpenMP construct.
124 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
125
126 /// Get the name of the capture helper.
127 StringRef getHelperName() const override { return HelperName; }
128
129 static bool classof(const CGCapturedStmtInfo *Info) {
130 return CGOpenMPRegionInfo::classof(Info) &&
131 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
132 ParallelOutlinedRegion;
133 }
134
135private:
136 /// A variable or parameter storing global thread id for OpenMP
137 /// constructs.
138 const VarDecl *ThreadIDVar;
139 StringRef HelperName;
140};
141
142/// API for captured statement code generation in OpenMP constructs.
143class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
144public:
145 class UntiedTaskActionTy final : public PrePostActionTy {
146 bool Untied;
147 const VarDecl *PartIDVar;
148 const RegionCodeGenTy UntiedCodeGen;
149 llvm::SwitchInst *UntiedSwitch = nullptr;
150
151 public:
152 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
153 const RegionCodeGenTy &UntiedCodeGen)
154 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
155 void Enter(CodeGenFunction &CGF) override {
156 if (Untied) {
157 // Emit task switching point.
158 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
159 CGF.GetAddrOfLocalVar(PartIDVar),
160 PartIDVar->getType()->castAs<PointerType>());
161 llvm::Value *Res =
162 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
163 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
164 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
165 CGF.EmitBlock(DoneBB);
166 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
167 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
168 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
169 CGF.Builder.GetInsertBlock());
170 emitUntiedSwitch(CGF);
171 }
172 }
173 void emitUntiedSwitch(CodeGenFunction &CGF) const {
174 if (Untied) {
175 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
176 CGF.GetAddrOfLocalVar(PartIDVar),
177 PartIDVar->getType()->castAs<PointerType>());
178 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
179 PartIdLVal);
180 UntiedCodeGen(CGF);
181 CodeGenFunction::JumpDest CurPoint =
182 CGF.getJumpDestInCurrentScope(".untied.next.");
183 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
184 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
185 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
186 CGF.Builder.GetInsertBlock());
187 CGF.EmitBranchThroughCleanup(CurPoint);
188 CGF.EmitBlock(CurPoint.getBlock());
189 }
190 }
191 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
192 };
193 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
194 const VarDecl *ThreadIDVar,
195 const RegionCodeGenTy &CodeGen,
196 OpenMPDirectiveKind Kind, bool HasCancel,
197 const UntiedTaskActionTy &Action)
198 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
199 ThreadIDVar(ThreadIDVar), Action(Action) {
200 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 200, __PRETTY_FUNCTION__))
;
201 }
202
203 /// Get a variable or parameter for storing global thread id
204 /// inside OpenMP construct.
205 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
206
207 /// Get an LValue for the current ThreadID variable.
208 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
209
210 /// Get the name of the capture helper.
211 StringRef getHelperName() const override { return ".omp_outlined."; }
212
213 void emitUntiedSwitch(CodeGenFunction &CGF) override {
214 Action.emitUntiedSwitch(CGF);
215 }
216
217 static bool classof(const CGCapturedStmtInfo *Info) {
218 return CGOpenMPRegionInfo::classof(Info) &&
219 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
220 TaskOutlinedRegion;
221 }
222
223private:
224 /// A variable or parameter storing global thread id for OpenMP
225 /// constructs.
226 const VarDecl *ThreadIDVar;
227 /// Action for emitting code for untied tasks.
228 const UntiedTaskActionTy &Action;
229};
230
231/// API for inlined captured statement code generation in OpenMP
232/// constructs.
233class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
234public:
235 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
236 const RegionCodeGenTy &CodeGen,
237 OpenMPDirectiveKind Kind, bool HasCancel)
238 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
239 OldCSI(OldCSI),
240 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
241
242 // Retrieve the value of the context parameter.
243 llvm::Value *getContextValue() const override {
244 if (OuterRegionInfo)
245 return OuterRegionInfo->getContextValue();
246 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 246)
;
247 }
248
249 void setContextValue(llvm::Value *V) override {
250 if (OuterRegionInfo) {
251 OuterRegionInfo->setContextValue(V);
252 return;
253 }
254 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 254)
;
255 }
256
257 /// Lookup the captured field decl for a variable.
258 const FieldDecl *lookup(const VarDecl *VD) const override {
259 if (OuterRegionInfo)
260 return OuterRegionInfo->lookup(VD);
261 // If there is no outer outlined region,no need to lookup in a list of
262 // captured variables, we can use the original one.
263 return nullptr;
264 }
265
266 FieldDecl *getThisFieldDecl() const override {
267 if (OuterRegionInfo)
268 return OuterRegionInfo->getThisFieldDecl();
269 return nullptr;
270 }
271
272 /// Get a variable or parameter for storing global thread id
273 /// inside OpenMP construct.
274 const VarDecl *getThreadIDVariable() const override {
275 if (OuterRegionInfo)
276 return OuterRegionInfo->getThreadIDVariable();
277 return nullptr;
278 }
279
280 /// Get an LValue for the current ThreadID variable.
281 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
284 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 284)
;
285 }
286
287 /// Get the name of the capture helper.
288 StringRef getHelperName() const override {
289 if (auto *OuterRegionInfo = getOldCSI())
290 return OuterRegionInfo->getHelperName();
291 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 291)
;
292 }
293
294 void emitUntiedSwitch(CodeGenFunction &CGF) override {
295 if (OuterRegionInfo)
296 OuterRegionInfo->emitUntiedSwitch(CGF);
297 }
298
299 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
300
301 static bool classof(const CGCapturedStmtInfo *Info) {
302 return CGOpenMPRegionInfo::classof(Info) &&
303 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
304 }
305
306 ~CGOpenMPInlinedRegionInfo() override = default;
307
308private:
309 /// CodeGen info about outer OpenMP region.
310 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
311 CGOpenMPRegionInfo *OuterRegionInfo;
312};
313
314/// API for captured statement code generation in OpenMP target
315/// constructs. For this captures, implicit parameters are used instead of the
316/// captured fields. The name of the target region has to be unique in a given
317/// application so it is provided by the client, because only the client has
318/// the information to generate that.
319class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
320public:
321 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
322 const RegionCodeGenTy &CodeGen, StringRef HelperName)
323 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
324 /*HasCancel=*/false),
325 HelperName(HelperName) {}
326
327 /// This is unused for target regions because each starts executing
328 /// with a single thread.
329 const VarDecl *getThreadIDVariable() const override { return nullptr; }
330
331 /// Get the name of the capture helper.
332 StringRef getHelperName() const override { return HelperName; }
333
334 static bool classof(const CGCapturedStmtInfo *Info) {
335 return CGOpenMPRegionInfo::classof(Info) &&
336 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
337 }
338
339private:
340 StringRef HelperName;
341};
342
343static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
344 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 344)
;
345}
346/// API for generation of expressions captured in a innermost OpenMP
347/// region.
348class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
349public:
350 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
351 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
352 OMPD_unknown,
353 /*HasCancel=*/false),
354 PrivScope(CGF) {
355 // Make sure the globals captured in the provided statement are local by
356 // using the privatization logic. We assume the same variable is not
357 // captured more than once.
358 for (const auto &C : CS.captures()) {
359 if (!C.capturesVariable() && !C.capturesVariableByCopy())
360 continue;
361
362 const VarDecl *VD = C.getCapturedVar();
363 if (VD->isLocalVarDeclOrParm())
364 continue;
365
366 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
367 /*RefersToEnclosingVariableOrCapture=*/false,
368 VD->getType().getNonReferenceType(), VK_LValue,
369 C.getLocation());
370 PrivScope.addPrivate(
371 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
372 }
373 (void)PrivScope.Privatize();
374 }
375
376 /// Lookup the captured field decl for a variable.
377 const FieldDecl *lookup(const VarDecl *VD) const override {
378 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
379 return FD;
380 return nullptr;
381 }
382
383 /// Emit the captured statement body.
384 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
385 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 385)
;
386 }
387
388 /// Get a variable or parameter for storing global thread id
389 /// inside OpenMP construct.
390 const VarDecl *getThreadIDVariable() const override {
391 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 391)
;
392 }
393
394 /// Get the name of the capture helper.
395 StringRef getHelperName() const override {
396 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 396)
;
397 }
398
399 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
400
401private:
402 /// Private scope to capture global variables.
403 CodeGenFunction::OMPPrivateScope PrivScope;
404};
405
406/// RAII for emitting code of OpenMP constructs.
407class InlinedOpenMPRegionRAII {
408 CodeGenFunction &CGF;
409 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
410 FieldDecl *LambdaThisCaptureField = nullptr;
411 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
412
413public:
414 /// Constructs region for combined constructs.
415 /// \param CodeGen Code generation sequence for combined directives. Includes
416 /// a list of functions used for code generation of implicitly inlined
417 /// regions.
418 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
419 OpenMPDirectiveKind Kind, bool HasCancel)
420 : CGF(CGF) {
421 // Start emission for the construct.
422 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
423 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
424 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
425 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
426 CGF.LambdaThisCaptureField = nullptr;
427 BlockInfo = CGF.BlockInfo;
428 CGF.BlockInfo = nullptr;
429 }
430
431 ~InlinedOpenMPRegionRAII() {
432 // Restore original CapturedStmtInfo only if we're done with code emission.
433 auto *OldCSI =
434 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
435 delete CGF.CapturedStmtInfo;
436 CGF.CapturedStmtInfo = OldCSI;
437 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
438 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
439 CGF.BlockInfo = BlockInfo;
440 }
441};
442
443/// Values for bit flags used in the ident_t to describe the fields.
444/// All enumeric elements are named and described in accordance with the code
445/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
446enum OpenMPLocationFlags : unsigned {
447 /// Use trampoline for internal microtask.
448 OMP_IDENT_IMD = 0x01,
449 /// Use c-style ident structure.
450 OMP_IDENT_KMPC = 0x02,
451 /// Atomic reduction option for kmpc_reduce.
452 OMP_ATOMIC_REDUCE = 0x10,
453 /// Explicit 'barrier' directive.
454 OMP_IDENT_BARRIER_EXPL = 0x20,
455 /// Implicit barrier in code.
456 OMP_IDENT_BARRIER_IMPL = 0x40,
457 /// Implicit barrier in 'for' directive.
458 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
459 /// Implicit barrier in 'sections' directive.
460 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
461 /// Implicit barrier in 'single' directive.
462 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
463 /// Call of __kmp_for_static_init for static loop.
464 OMP_IDENT_WORK_LOOP = 0x200,
465 /// Call of __kmp_for_static_init for sections.
466 OMP_IDENT_WORK_SECTIONS = 0x400,
467 /// Call of __kmp_for_static_init for distribute.
468 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
469 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
470};
471
472namespace {
473LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
474/// Values for bit flags for marking which requires clauses have been used.
475enum OpenMPOffloadingRequiresDirFlags : int64_t {
476 /// flag undefined.
477 OMP_REQ_UNDEFINED = 0x000,
478 /// no requires clause present.
479 OMP_REQ_NONE = 0x001,
480 /// reverse_offload clause.
481 OMP_REQ_REVERSE_OFFLOAD = 0x002,
482 /// unified_address clause.
483 OMP_REQ_UNIFIED_ADDRESS = 0x004,
484 /// unified_shared_memory clause.
485 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
486 /// dynamic_allocators clause.
487 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
488 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
489};
490
491enum OpenMPOffloadingReservedDeviceIDs {
492 /// Device ID if the device was not defined, runtime should get it
493 /// from environment variables in the spec.
494 OMP_DEVICEID_UNDEF = -1,
495};
496} // anonymous namespace
497
498/// Describes ident structure that describes a source location.
499/// All descriptions are taken from
500/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
501/// Original structure:
502/// typedef struct ident {
503/// kmp_int32 reserved_1; /**< might be used in Fortran;
504/// see above */
505/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
506/// KMP_IDENT_KMPC identifies this union
507/// member */
508/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
509/// see above */
510///#if USE_ITT_BUILD
511/// /* but currently used for storing
512/// region-specific ITT */
513/// /* contextual information. */
514///#endif /* USE_ITT_BUILD */
515/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
516/// C++ */
517/// char const *psource; /**< String describing the source location.
518/// The string is composed of semi-colon separated
519// fields which describe the source file,
520/// the function and a pair of line numbers that
521/// delimit the construct.
522/// */
523/// } ident_t;
524enum IdentFieldIndex {
525 /// might be used in Fortran
526 IdentField_Reserved_1,
527 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
528 IdentField_Flags,
529 /// Not really used in Fortran any more
530 IdentField_Reserved_2,
531 /// Source[4] in Fortran, do not use for C++
532 IdentField_Reserved_3,
533 /// String describing the source location. The string is composed of
534 /// semi-colon separated fields which describe the source file, the function
535 /// and a pair of line numbers that delimit the construct.
536 IdentField_PSource
537};
538
539/// Schedule types for 'omp for' loops (these enumerators are taken from
540/// the enum sched_type in kmp.h).
541enum OpenMPSchedType {
542 /// Lower bound for default (unordered) versions.
543 OMP_sch_lower = 32,
544 OMP_sch_static_chunked = 33,
545 OMP_sch_static = 34,
546 OMP_sch_dynamic_chunked = 35,
547 OMP_sch_guided_chunked = 36,
548 OMP_sch_runtime = 37,
549 OMP_sch_auto = 38,
550 /// static with chunk adjustment (e.g., simd)
551 OMP_sch_static_balanced_chunked = 45,
552 /// Lower bound for 'ordered' versions.
553 OMP_ord_lower = 64,
554 OMP_ord_static_chunked = 65,
555 OMP_ord_static = 66,
556 OMP_ord_dynamic_chunked = 67,
557 OMP_ord_guided_chunked = 68,
558 OMP_ord_runtime = 69,
559 OMP_ord_auto = 70,
560 OMP_sch_default = OMP_sch_static,
561 /// dist_schedule types
562 OMP_dist_sch_static_chunked = 91,
563 OMP_dist_sch_static = 92,
564 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
565 /// Set if the monotonic schedule modifier was present.
566 OMP_sch_modifier_monotonic = (1 << 29),
567 /// Set if the nonmonotonic schedule modifier was present.
568 OMP_sch_modifier_nonmonotonic = (1 << 30),
569};
570
571/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
572/// region.
573class CleanupTy final : public EHScopeStack::Cleanup {
574 PrePostActionTy *Action;
575
576public:
577 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
578 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
579 if (!CGF.HaveInsertPoint())
580 return;
581 Action->Exit(CGF);
582 }
583};
584
585} // anonymous namespace
586
587void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
588 CodeGenFunction::RunCleanupsScope Scope(CGF);
589 if (PrePostAction
13.1
Field 'PrePostAction' is null
13.1
Field 'PrePostAction' is null
) {
14
Taking false branch
590 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
591 Callback(CodeGen, CGF, *PrePostAction);
592 } else {
593 PrePostActionTy Action;
594 Callback(CodeGen, CGF, Action);
15
Calling 'RegionCodeGenTy::CallbackFn'
595 }
596}
597
598/// Check if the combiner is a call to UDR combiner and if it is so return the
599/// UDR decl used for reduction.
600static const OMPDeclareReductionDecl *
601getReductionInit(const Expr *ReductionOp) {
602 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
603 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
604 if (const auto *DRE =
605 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
606 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
607 return DRD;
608 return nullptr;
609}
610
611static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
612 const OMPDeclareReductionDecl *DRD,
613 const Expr *InitOp,
614 Address Private, Address Original,
615 QualType Ty) {
616 if (DRD->getInitializer()) {
617 std::pair<llvm::Function *, llvm::Function *> Reduction =
618 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
619 const auto *CE = cast<CallExpr>(InitOp);
620 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
621 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
622 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
623 const auto *LHSDRE =
624 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
625 const auto *RHSDRE =
626 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
627 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
628 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
629 [=]() { return Private; });
630 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
631 [=]() { return Original; });
632 (void)PrivateScope.Privatize();
633 RValue Func = RValue::get(Reduction.second);
634 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
635 CGF.EmitIgnoredExpr(InitOp);
636 } else {
637 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
638 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
639 auto *GV = new llvm::GlobalVariable(
640 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
641 llvm::GlobalValue::PrivateLinkage, Init, Name);
642 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
643 RValue InitRVal;
644 switch (CGF.getEvaluationKind(Ty)) {
645 case TEK_Scalar:
646 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
647 break;
648 case TEK_Complex:
649 InitRVal =
650 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
651 break;
652 case TEK_Aggregate:
653 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
654 break;
655 }
656 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
657 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
658 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
659 /*IsInitializer=*/false);
660 }
661}
662
663/// Emit initialization of arrays of complex types.
664/// \param DestAddr Address of the array.
665/// \param Type Type of array.
666/// \param Init Initial expression of array.
667/// \param SrcAddr Address of the original array.
668static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
669 QualType Type, bool EmitDeclareReductionInit,
670 const Expr *Init,
671 const OMPDeclareReductionDecl *DRD,
672 Address SrcAddr = Address::invalid()) {
673 // Perform element-by-element initialization.
674 QualType ElementTy;
675
676 // Drill down to the base element type on both arrays.
677 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
678 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
679 DestAddr =
680 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
681 if (DRD)
682 SrcAddr =
683 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
684
685 llvm::Value *SrcBegin = nullptr;
686 if (DRD)
687 SrcBegin = SrcAddr.getPointer();
688 llvm::Value *DestBegin = DestAddr.getPointer();
689 // Cast from pointer to array type to pointer to single element.
690 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
691 // The basic structure here is a while-do loop.
692 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
693 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
694 llvm::Value *IsEmpty =
695 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
696 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
697
698 // Enter the loop body, making that address the current address.
699 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
700 CGF.EmitBlock(BodyBB);
701
702 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
703
704 llvm::PHINode *SrcElementPHI = nullptr;
705 Address SrcElementCurrent = Address::invalid();
706 if (DRD) {
707 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
708 "omp.arraycpy.srcElementPast");
709 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
710 SrcElementCurrent =
711 Address(SrcElementPHI,
712 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
713 }
714 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
715 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
716 DestElementPHI->addIncoming(DestBegin, EntryBB);
717 Address DestElementCurrent =
718 Address(DestElementPHI,
719 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
720
721 // Emit copy.
722 {
723 CodeGenFunction::RunCleanupsScope InitScope(CGF);
724 if (EmitDeclareReductionInit) {
725 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
726 SrcElementCurrent, ElementTy);
727 } else
728 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
729 /*IsInitializer=*/false);
730 }
731
732 if (DRD) {
733 // Shift the address forward by one element.
734 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
735 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
736 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
737 }
738
739 // Shift the address forward by one element.
740 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
741 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
742 // Check whether we've reached the end.
743 llvm::Value *Done =
744 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
745 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
746 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
747
748 // Done.
749 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
750}
751
752LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
753 return CGF.EmitOMPSharedLValue(E);
754}
755
756LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
757 const Expr *E) {
758 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
759 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
760 return LValue();
761}
762
763void ReductionCodeGen::emitAggregateInitialization(
764 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
765 const OMPDeclareReductionDecl *DRD) {
766 // Emit VarDecl with copy init for arrays.
767 // Get the address of the original variable captured in current
768 // captured region.
769 const auto *PrivateVD =
770 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
771 bool EmitDeclareReductionInit =
772 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
773 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
774 EmitDeclareReductionInit,
775 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
776 : PrivateVD->getInit(),
777 DRD, SharedLVal.getAddress(CGF));
778}
779
780ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
781 ArrayRef<const Expr *> Origs,
782 ArrayRef<const Expr *> Privates,
783 ArrayRef<const Expr *> ReductionOps) {
784 ClausesData.reserve(Shareds.size());
785 SharedAddresses.reserve(Shareds.size());
786 Sizes.reserve(Shareds.size());
787 BaseDecls.reserve(Shareds.size());
788 const auto *IOrig = Origs.begin();
789 const auto *IPriv = Privates.begin();
790 const auto *IRed = ReductionOps.begin();
791 for (const Expr *Ref : Shareds) {
792 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
793 std::advance(IOrig, 1);
794 std::advance(IPriv, 1);
795 std::advance(IRed, 1);
796 }
797}
798
799void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
800 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
801 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
;
802 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
803 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
804 SharedAddresses.emplace_back(First, Second);
805 if (ClausesData[N].Shared == ClausesData[N].Ref) {
806 OrigAddresses.emplace_back(First, Second);
807 } else {
808 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
809 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
810 OrigAddresses.emplace_back(First, Second);
811 }
812}
813
814void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
815 const auto *PrivateVD =
816 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
817 QualType PrivateType = PrivateVD->getType();
818 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
819 if (!PrivateType->isVariablyModifiedType()) {
820 Sizes.emplace_back(
821 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
822 nullptr);
823 return;
824 }
825 llvm::Value *Size;
826 llvm::Value *SizeInChars;
827 auto *ElemType =
828 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType())
829 ->getElementType();
830 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
831 if (AsArraySection) {
832 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
833 OrigAddresses[N].first.getPointer(CGF));
834 Size = CGF.Builder.CreateNUWAdd(
835 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
836 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
837 } else {
838 SizeInChars =
839 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
840 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
841 }
842 Sizes.emplace_back(SizeInChars, Size);
843 CodeGenFunction::OpaqueValueMapping OpaqueMap(
844 CGF,
845 cast<OpaqueValueExpr>(
846 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
847 RValue::get(Size));
848 CGF.EmitVariablyModifiedType(PrivateType);
849}
850
851void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
852 llvm::Value *Size) {
853 const auto *PrivateVD =
854 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
855 QualType PrivateType = PrivateVD->getType();
856 if (!PrivateType->isVariablyModifiedType()) {
857 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
858 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
859 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
;
860 return;
861 }
862 CodeGenFunction::OpaqueValueMapping OpaqueMap(
863 CGF,
864 cast<OpaqueValueExpr>(
865 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
866 RValue::get(Size));
867 CGF.EmitVariablyModifiedType(PrivateType);
868}
869
870void ReductionCodeGen::emitInitialization(
871 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
872 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
873 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 873, __PRETTY_FUNCTION__))
;
874 const auto *PrivateVD =
875 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
876 const OMPDeclareReductionDecl *DRD =
877 getReductionInit(ClausesData[N].ReductionOp);
878 QualType PrivateType = PrivateVD->getType();
879 PrivateAddr = CGF.Builder.CreateElementBitCast(
880 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
881 QualType SharedType = SharedAddresses[N].first.getType();
882 SharedLVal = CGF.MakeAddrLValue(
883 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
884 CGF.ConvertTypeForMem(SharedType)),
885 SharedType, SharedAddresses[N].first.getBaseInfo(),
886 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
887 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
888 if (DRD && DRD->getInitializer())
889 (void)DefaultInit(CGF);
890 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
891 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
892 (void)DefaultInit(CGF);
893 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
894 PrivateAddr, SharedLVal.getAddress(CGF),
895 SharedLVal.getType());
896 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
897 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
898 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
899 PrivateVD->getType().getQualifiers(),
900 /*IsInitializer=*/false);
901 }
902}
903
904bool ReductionCodeGen::needCleanups(unsigned N) {
905 const auto *PrivateVD =
906 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
907 QualType PrivateType = PrivateVD->getType();
908 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
909 return DTorKind != QualType::DK_none;
910}
911
912void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
913 Address PrivateAddr) {
914 const auto *PrivateVD =
915 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
916 QualType PrivateType = PrivateVD->getType();
917 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
918 if (needCleanups(N)) {
919 PrivateAddr = CGF.Builder.CreateElementBitCast(
920 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
921 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
922 }
923}
924
925static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
926 LValue BaseLV) {
927 BaseTy = BaseTy.getNonReferenceType();
928 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
929 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
930 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
931 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
932 } else {
933 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
934 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
935 }
936 BaseTy = BaseTy->getPointeeType();
937 }
938 return CGF.MakeAddrLValue(
939 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
940 CGF.ConvertTypeForMem(ElTy)),
941 BaseLV.getType(), BaseLV.getBaseInfo(),
942 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
943}
944
945static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
946 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
947 llvm::Value *Addr) {
948 Address Tmp = Address::invalid();
949 Address TopTmp = Address::invalid();
950 Address MostTopTmp = Address::invalid();
951 BaseTy = BaseTy.getNonReferenceType();
952 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
953 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
954 Tmp = CGF.CreateMemTemp(BaseTy);
955 if (TopTmp.isValid())
956 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
957 else
958 MostTopTmp = Tmp;
959 TopTmp = Tmp;
960 BaseTy = BaseTy->getPointeeType();
961 }
962 llvm::Type *Ty = BaseLVType;
963 if (Tmp.isValid())
964 Ty = Tmp.getElementType();
965 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
966 if (Tmp.isValid()) {
967 CGF.Builder.CreateStore(Addr, Tmp);
968 return MostTopTmp;
969 }
970 return Address(Addr, BaseLVAlignment);
971}
972
973static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
974 const VarDecl *OrigVD = nullptr;
975 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
976 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
977 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
978 Base = TempOASE->getBase()->IgnoreParenImpCasts();
979 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
980 Base = TempASE->getBase()->IgnoreParenImpCasts();
981 DE = cast<DeclRefExpr>(Base);
982 OrigVD = cast<VarDecl>(DE->getDecl());
983 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
984 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
985 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
986 Base = TempASE->getBase()->IgnoreParenImpCasts();
987 DE = cast<DeclRefExpr>(Base);
988 OrigVD = cast<VarDecl>(DE->getDecl());
989 }
990 return OrigVD;
991}
992
993Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
994 Address PrivateAddr) {
995 const DeclRefExpr *DE;
996 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
997 BaseDecls.emplace_back(OrigVD);
998 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
999 LValue BaseLValue =
1000 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1001 OriginalBaseLValue);
1002 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1003 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1004 llvm::Value *PrivatePointer =
1005 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1006 PrivateAddr.getPointer(),
1007 SharedAddresses[N].first.getAddress(CGF).getType());
1008 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1009 return castToBase(CGF, OrigVD->getType(),
1010 SharedAddresses[N].first.getType(),
1011 OriginalBaseLValue.getAddress(CGF).getType(),
1012 OriginalBaseLValue.getAlignment(), Ptr);
1013 }
1014 BaseDecls.emplace_back(
1015 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1016 return PrivateAddr;
1017}
1018
1019bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1020 const OMPDeclareReductionDecl *DRD =
1021 getReductionInit(ClausesData[N].ReductionOp);
1022 return DRD && DRD->getInitializer();
1023}
1024
1025LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1026 return CGF.EmitLoadOfPointerLValue(
1027 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1028 getThreadIDVariable()->getType()->castAs<PointerType>());
1029}
1030
1031void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1032 if (!CGF.HaveInsertPoint())
1033 return;
1034 // 1.2.2 OpenMP Language Terminology
1035 // Structured block - An executable statement with a single entry at the
1036 // top and a single exit at the bottom.
1037 // The point of exit cannot be a branch out of the structured block.
1038 // longjmp() and throw() must not violate the entry/exit criteria.
1039 CGF.EHStack.pushTerminate();
1040 CodeGen(CGF);
1041 CGF.EHStack.popTerminate();
1042}
1043
1044LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1045 CodeGenFunction &CGF) {
1046 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1047 getThreadIDVariable()->getType(),
1048 AlignmentSource::Decl);
1049}
1050
1051static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1052 QualType FieldTy) {
1053 auto *Field = FieldDecl::Create(
1054 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1055 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1056 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1057 Field->setAccess(AS_public);
1058 DC->addDecl(Field);
1059 return Field;
1060}
1061
1062CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1063 StringRef Separator)
1064 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1065 OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
1066 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1067
1068 // Initialize Types used in OpenMPIRBuilder from OMPKinds.def
1069 OMPBuilder.initialize();
1070 loadOffloadInfoMetadata();
1071}
1072
1073void CGOpenMPRuntime::clear() {
1074 InternalVars.clear();
1075 // Clean non-target variable declarations possibly used only in debug info.
1076 for (const auto &Data : EmittedNonTargetVariables) {
1077 if (!Data.getValue().pointsToAliveValue())
1078 continue;
1079 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1080 if (!GV)
1081 continue;
1082 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1083 continue;
1084 GV->eraseFromParent();
1085 }
1086}
1087
1088std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1089 SmallString<128> Buffer;
1090 llvm::raw_svector_ostream OS(Buffer);
1091 StringRef Sep = FirstSeparator;
1092 for (StringRef Part : Parts) {
1093 OS << Sep << Part;
1094 Sep = Separator;
1095 }
1096 return std::string(OS.str());
1097}
1098
1099static llvm::Function *
1100emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1101 const Expr *CombinerInitializer, const VarDecl *In,
1102 const VarDecl *Out, bool IsCombiner) {
1103 // void .omp_combiner.(Ty *in, Ty *out);
1104 ASTContext &C = CGM.getContext();
1105 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1106 FunctionArgList Args;
1107 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1108 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1109 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1110 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1111 Args.push_back(&OmpOutParm);
1112 Args.push_back(&OmpInParm);
1113 const CGFunctionInfo &FnInfo =
1114 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1115 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1116 std::string Name = CGM.getOpenMPRuntime().getName(
1117 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1118 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1119 Name, &CGM.getModule());
1120 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1121 if (CGM.getLangOpts().Optimize) {
1122 Fn->removeFnAttr(llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1125 }
1126 CodeGenFunction CGF(CGM);
1127 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1128 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1129 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1130 Out->getLocation());
1131 CodeGenFunction::OMPPrivateScope Scope(CGF);
1132 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1133 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1134 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1135 .getAddress(CGF);
1136 });
1137 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1138 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1139 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1140 .getAddress(CGF);
1141 });
1142 (void)Scope.Privatize();
1143 if (!IsCombiner && Out->hasInit() &&
1144 !CGF.isTrivialInitializer(Out->getInit())) {
1145 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1146 Out->getType().getQualifiers(),
1147 /*IsInitializer=*/true);
1148 }
1149 if (CombinerInitializer)
1150 CGF.EmitIgnoredExpr(CombinerInitializer);
1151 Scope.ForceCleanup();
1152 CGF.FinishFunction();
1153 return Fn;
1154}
1155
1156void CGOpenMPRuntime::emitUserDefinedReduction(
1157 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1158 if (UDRMap.count(D) > 0)
1159 return;
1160 llvm::Function *Combiner = emitCombinerOrInitializer(
1161 CGM, D->getType(), D->getCombiner(),
1162 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1163 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1164 /*IsCombiner=*/true);
1165 llvm::Function *Initializer = nullptr;
1166 if (const Expr *Init = D->getInitializer()) {
1167 Initializer = emitCombinerOrInitializer(
1168 CGM, D->getType(),
1169 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1170 : nullptr,
1171 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1172 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1173 /*IsCombiner=*/false);
1174 }
1175 UDRMap.try_emplace(D, Combiner, Initializer);
1176 if (CGF) {
1177 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1178 Decls.second.push_back(D);
1179 }
1180}
1181
1182std::pair<llvm::Function *, llvm::Function *>
1183CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1184 auto I = UDRMap.find(D);
1185 if (I != UDRMap.end())
1186 return I->second;
1187 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1188 return UDRMap.lookup(D);
1189}
1190
1191namespace {
1192// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1193// Builder if one is present.
1194struct PushAndPopStackRAII {
1195 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1196 bool HasCancel)
1197 : OMPBuilder(OMPBuilder) {
1198 if (!OMPBuilder)
1199 return;
1200
1201 // The following callback is the crucial part of clangs cleanup process.
1202 //
1203 // NOTE:
1204 // Once the OpenMPIRBuilder is used to create parallel regions (and
1205 // similar), the cancellation destination (Dest below) is determined via
1206 // IP. That means if we have variables to finalize we split the block at IP,
1207 // use the new block (=BB) as destination to build a JumpDest (via
1208 // getJumpDestInCurrentScope(BB)) which then is fed to
1209 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1210 // to push & pop an FinalizationInfo object.
1211 // The FiniCB will still be needed but at the point where the
1212 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1213 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1214 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
1215 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
;
1216 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1217 CGF.Builder.restoreIP(IP);
1218 CodeGenFunction::JumpDest Dest =
1219 CGF.getOMPCancelDestination(OMPD_parallel);
1220 CGF.EmitBranchThroughCleanup(Dest);
1221 };
1222
1223 // TODO: Remove this once we emit parallel regions through the
1224 // OpenMPIRBuilder as it can do this setup internally.
1225 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1226 {FiniCB, OMPD_parallel, HasCancel});
1227 OMPBuilder->pushFinalizationCB(std::move(FI));
1228 }
1229 ~PushAndPopStackRAII() {
1230 if (OMPBuilder)
1231 OMPBuilder->popFinalizationCB();
1232 }
1233 llvm::OpenMPIRBuilder *OMPBuilder;
1234};
1235} // namespace
1236
1237static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1238 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1239 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1240 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1241 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
1242 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
;
1243 CodeGenFunction CGF(CGM, true);
1244 bool HasCancel = false;
1245 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1246 HasCancel = OPD->hasCancel();
1247 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1248 HasCancel = OPD->hasCancel();
1249 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1250 HasCancel = OPSD->hasCancel();
1251 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1254 HasCancel = OPFD->hasCancel();
1255 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1256 HasCancel = OPFD->hasCancel();
1257 else if (const auto *OPFD =
1258 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1259 HasCancel = OPFD->hasCancel();
1260 else if (const auto *OPFD =
1261 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1262 HasCancel = OPFD->hasCancel();
1263
1264 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1265 // parallel region to make cancellation barriers work properly.
1266 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1267 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel);
1268 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1269 HasCancel, OutlinedHelperName);
1270 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1271 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1272}
1273
1274llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1275 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1276 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1277 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1278 return emitParallelOrTeamsOutlinedFunction(
1279 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1280}
1281
1282llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1283 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1284 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1285 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1286 return emitParallelOrTeamsOutlinedFunction(
1287 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1288}
1289
1290llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1291 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1292 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1293 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1294 bool Tied, unsigned &NumberOfParts) {
1295 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1296 PrePostActionTy &) {
1297 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1298 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1299 llvm::Value *TaskArgs[] = {
1300 UpLoc, ThreadID,
1301 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1302 TaskTVar->getType()->castAs<PointerType>())
1303 .getPointer(CGF)};
1304 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1305 CGM.getModule(), OMPRTL___kmpc_omp_task),
1306 TaskArgs);
1307 };
1308 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1309 UntiedCodeGen);
1310 CodeGen.setAction(Action);
1311 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
1312 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
;
1313 const OpenMPDirectiveKind Region =
1314 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1315 : OMPD_task;
1316 const CapturedStmt *CS = D.getCapturedStmt(Region);
1317 bool HasCancel = false;
1318 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1319 HasCancel = TD->hasCancel();
1320 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1321 HasCancel = TD->hasCancel();
1322 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1323 HasCancel = TD->hasCancel();
1324 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1325 HasCancel = TD->hasCancel();
1326
1327 CodeGenFunction CGF(CGM, true);
1328 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1329 InnermostKind, HasCancel, Action);
1330 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1331 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1332 if (!Tied)
1333 NumberOfParts = Action.getNumberOfParts();
1334 return Res;
1335}
1336
1337static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1338 const RecordDecl *RD, const CGRecordLayout &RL,
1339 ArrayRef<llvm::Constant *> Data) {
1340 llvm::StructType *StructTy = RL.getLLVMType();
1341 unsigned PrevIdx = 0;
1342 ConstantInitBuilder CIBuilder(CGM);
1343 auto DI = Data.begin();
1344 for (const FieldDecl *FD : RD->fields()) {
1345 unsigned Idx = RL.getLLVMFieldNo(FD);
1346 // Fill the alignment.
1347 for (unsigned I = PrevIdx; I < Idx; ++I)
1348 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1349 PrevIdx = Idx + 1;
1350 Fields.add(*DI);
1351 ++DI;
1352 }
1353}
1354
1355template <class... As>
1356static llvm::GlobalVariable *
1357createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1358 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1359 As &&... Args) {
1360 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1361 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1362 ConstantInitBuilder CIBuilder(CGM);
1363 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1364 buildStructValue(Fields, CGM, RD, RL, Data);
1365 return Fields.finishAndCreateGlobal(
1366 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1367 std::forward<As>(Args)...);
1368}
1369
1370template <typename T>
1371static void
1372createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1373 ArrayRef<llvm::Constant *> Data,
1374 T &Parent) {
1375 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1376 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1377 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1378 buildStructValue(Fields, CGM, RD, RL, Data);
1379 Fields.finishAndAddTo(Parent);
1380}
1381
1382void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1383 bool AtCurrentPoint) {
1384 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1385 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1385, __PRETTY_FUNCTION__))
;
1386
1387 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1388 if (AtCurrentPoint) {
1389 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1390 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1391 } else {
1392 Elem.second.ServiceInsertPt =
1393 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1394 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1395 }
1396}
1397
1398void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1399 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1400 if (Elem.second.ServiceInsertPt) {
1401 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1402 Elem.second.ServiceInsertPt = nullptr;
1403 Ptr->eraseFromParent();
1404 }
1405}
1406
1407static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1408 SourceLocation Loc,
1409 SmallString<128> &Buffer) {
1410 llvm::raw_svector_ostream OS(Buffer);
1411 // Build debug location
1412 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1413 OS << ";" << PLoc.getFilename() << ";";
1414 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1415 OS << FD->getQualifiedNameAsString();
1416 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1417 return OS.str();
1418}
1419
1420llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1421 SourceLocation Loc,
1422 unsigned Flags) {
1423 llvm::Constant *SrcLocStr;
1424 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1425 Loc.isInvalid()) {
1426 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr();
1427 } else {
1428 std::string FunctionName = "";
1429 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1430 FunctionName = FD->getQualifiedNameAsString();
1431 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1432 const char *FileName = PLoc.getFilename();
1433 unsigned Line = PLoc.getLine();
1434 unsigned Column = PLoc.getColumn();
1435 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName.c_str(), FileName,
1436 Line, Column);
1437 }
1438 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1439 return OMPBuilder.getOrCreateIdent(SrcLocStr, llvm::omp::IdentFlag(Flags),
1440 Reserved2Flags);
1441}
1442
1443llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1444 SourceLocation Loc) {
1445 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1445, __PRETTY_FUNCTION__))
;
1446 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1447 // the clang invariants used below might be broken.
1448 if (CGM.getLangOpts().OpenMPIRBuilder) {
1449 SmallString<128> Buffer;
1450 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1451 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1452 getIdentStringFromSourceLocation(CGF, Loc, Buffer));
1453 return OMPBuilder.getOrCreateThreadID(
1454 OMPBuilder.getOrCreateIdent(SrcLocStr));
1455 }
1456
1457 llvm::Value *ThreadID = nullptr;
1458 // Check whether we've already cached a load of the thread id in this
1459 // function.
1460 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1461 if (I != OpenMPLocThreadIDMap.end()) {
1462 ThreadID = I->second.ThreadID;
1463 if (ThreadID != nullptr)
1464 return ThreadID;
1465 }
1466 // If exceptions are enabled, do not use parameter to avoid possible crash.
1467 if (auto *OMPRegionInfo =
1468 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1469 if (OMPRegionInfo->getThreadIDVariable()) {
1470 // Check if this an outlined function with thread id passed as argument.
1471 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1472 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1473 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1474 !CGF.getLangOpts().CXXExceptions ||
1475 CGF.Builder.GetInsertBlock() == TopBlock ||
1476 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1477 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1478 TopBlock ||
1479 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1480 CGF.Builder.GetInsertBlock()) {
1481 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1482 // If value loaded in entry block, cache it and use it everywhere in
1483 // function.
1484 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1485 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1486 Elem.second.ThreadID = ThreadID;
1487 }
1488 return ThreadID;
1489 }
1490 }
1491 }
1492
1493 // This is not an outlined function region - need to call __kmpc_int32
1494 // kmpc_global_thread_num(ident_t *loc).
1495 // Generate thread id value and cache this value for use across the
1496 // function.
1497 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1498 if (!Elem.second.ServiceInsertPt)
1499 setLocThreadIdInsertPt(CGF);
1500 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1501 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1502 llvm::CallInst *Call = CGF.Builder.CreateCall(
1503 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1504 OMPRTL___kmpc_global_thread_num),
1505 emitUpdateLocation(CGF, Loc));
1506 Call->setCallingConv(CGF.getRuntimeCC());
1507 Elem.second.ThreadID = Call;
1508 return Call;
1509}
1510
1511void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1512 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1512, __PRETTY_FUNCTION__))
;
1513 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1514 clearLocThreadIdInsertPt(CGF);
1515 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1516 }
1517 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1518 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1519 UDRMap.erase(D);
1520 FunctionUDRMap.erase(CGF.CurFn);
1521 }
1522 auto I = FunctionUDMMap.find(CGF.CurFn);
1523 if (I != FunctionUDMMap.end()) {
1524 for(const auto *D : I->second)
1525 UDMMap.erase(D);
1526 FunctionUDMMap.erase(I);
1527 }
1528 LastprivateConditionalToTypes.erase(CGF.CurFn);
1529 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1530}
1531
1532llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1533 return OMPBuilder.IdentPtr;
1534}
1535
1536llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1537 if (!Kmpc_MicroTy) {
1538 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1539 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1540 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1541 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1542 }
1543 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1544}
1545
1546llvm::FunctionCallee
1547CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
1548 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
1549 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
;
1550 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
1551 : "__kmpc_for_static_init_4u")
1552 : (IVSigned ? "__kmpc_for_static_init_8"
1553 : "__kmpc_for_static_init_8u");
1554 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1555 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1556 llvm::Type *TypeParams[] = {
1557 getIdentTyPointerTy(), // loc
1558 CGM.Int32Ty, // tid
1559 CGM.Int32Ty, // schedtype
1560 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1561 PtrTy, // p_lower
1562 PtrTy, // p_upper
1563 PtrTy, // p_stride
1564 ITy, // incr
1565 ITy // chunk
1566 };
1567 auto *FnTy =
1568 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1569 return CGM.CreateRuntimeFunction(FnTy, Name);
1570}
1571
1572llvm::FunctionCallee
1573CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
1574 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
1575 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
;
1576 StringRef Name =
1577 IVSize == 32
1578 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
1579 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
1580 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1581 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
1582 CGM.Int32Ty, // tid
1583 CGM.Int32Ty, // schedtype
1584 ITy, // lower
1585 ITy, // upper
1586 ITy, // stride
1587 ITy // chunk
1588 };
1589 auto *FnTy =
1590 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1591 return CGM.CreateRuntimeFunction(FnTy, Name);
1592}
1593
1594llvm::FunctionCallee
1595CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
1596 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
1597 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
;
1598 StringRef Name =
1599 IVSize == 32
1600 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
1601 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
1602 llvm::Type *TypeParams[] = {
1603 getIdentTyPointerTy(), // loc
1604 CGM.Int32Ty, // tid
1605 };
1606 auto *FnTy =
1607 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1608 return CGM.CreateRuntimeFunction(FnTy, Name);
1609}
1610
1611llvm::FunctionCallee
1612CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
1613 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
1614 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
;
1615 StringRef Name =
1616 IVSize == 32
1617 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
1618 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
1619 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1620 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1621 llvm::Type *TypeParams[] = {
1622 getIdentTyPointerTy(), // loc
1623 CGM.Int32Ty, // tid
1624 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1625 PtrTy, // p_lower
1626 PtrTy, // p_upper
1627 PtrTy // p_stride
1628 };
1629 auto *FnTy =
1630 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1631 return CGM.CreateRuntimeFunction(FnTy, Name);
1632}
1633
1634/// Obtain information that uniquely identifies a target entry. This
1635/// consists of the file and device IDs as well as line number associated with
1636/// the relevant entry source location.
1637static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
1638 unsigned &DeviceID, unsigned &FileID,
1639 unsigned &LineNum) {
1640 SourceManager &SM = C.getSourceManager();
1641
1642 // The loc should be always valid and have a file ID (the user cannot use
1643 // #pragma directives in macros)
1644
1645 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1645, __PRETTY_FUNCTION__))
;
1646
1647 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
1648 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1648, __PRETTY_FUNCTION__))
;
1649
1650 llvm::sys::fs::UniqueID ID;
1651 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
1652 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1653 << PLoc.getFilename() << EC.message();
1654
1655 DeviceID = ID.getDevice();
1656 FileID = ID.getFile();
1657 LineNum = PLoc.getLine();
1658}
1659
1660Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1661 if (CGM.getLangOpts().OpenMPSimd)
1662 return Address::invalid();
1663 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1664 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1665 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1666 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1667 HasRequiresUnifiedSharedMemory))) {
1668 SmallString<64> PtrName;
1669 {
1670 llvm::raw_svector_ostream OS(PtrName);
1671 OS << CGM.getMangledName(GlobalDecl(VD));
1672 if (!VD->isExternallyVisible()) {
1673 unsigned DeviceID, FileID, Line;
1674 getTargetEntryUniqueInfo(CGM.getContext(),
1675 VD->getCanonicalDecl()->getBeginLoc(),
1676 DeviceID, FileID, Line);
1677 OS << llvm::format("_%x", FileID);
1678 }
1679 OS << "_decl_tgt_ref_ptr";
1680 }
1681 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
1682 if (!Ptr) {
1683 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
1684 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
1685 PtrName);
1686
1687 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1688 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1689
1690 if (!CGM.getLangOpts().OpenMPIsDevice)
1691 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
1692 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
1693 }
1694 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
1695 }
1696 return Address::invalid();
1697}
1698
1699llvm::Constant *
1700CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1701 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
1702 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
;
1703 // Lookup the entry, lazily creating it if necessary.
1704 std::string Suffix = getName({"cache", ""});
1705 return getOrCreateInternalVariable(
1706 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
1707}
1708
1709Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1710 const VarDecl *VD,
1711 Address VDAddr,
1712 SourceLocation Loc) {
1713 if (CGM.getLangOpts().OpenMPUseTLS &&
1714 CGM.getContext().getTargetInfo().isTLSSupported())
1715 return VDAddr;
1716
1717 llvm::Type *VarTy = VDAddr.getElementType();
1718 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1719 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1720 CGM.Int8PtrTy),
1721 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1722 getOrCreateThreadPrivateCache(VD)};
1723 return Address(CGF.EmitRuntimeCall(
1724 OMPBuilder.getOrCreateRuntimeFunction(
1725 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1726 Args),
1727 VDAddr.getAlignment());
1728}
1729
1730void CGOpenMPRuntime::emitThreadPrivateVarInit(
1731 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1732 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1733 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1734 // library.
1735 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1736 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1737 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1738 OMPLoc);
1739 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1740 // to register constructor/destructor for variable.
1741 llvm::Value *Args[] = {
1742 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
1743 Ctor, CopyCtor, Dtor};
1744 CGF.EmitRuntimeCall(
1745 OMPBuilder.getOrCreateRuntimeFunction(
1746 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1747 Args);
1748}
1749
1750llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1751 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1752 bool PerformInit, CodeGenFunction *CGF) {
1753 if (CGM.getLangOpts().OpenMPUseTLS &&
1754 CGM.getContext().getTargetInfo().isTLSSupported())
1755 return nullptr;
1756
1757 VD = VD->getDefinition(CGM.getContext());
1758 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
1759 QualType ASTTy = VD->getType();
1760
1761 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1762 const Expr *Init = VD->getAnyInitializer();
1763 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1764 // Generate function that re-emits the declaration's initializer into the
1765 // threadprivate copy of the variable VD
1766 CodeGenFunction CtorCGF(CGM);
1767 FunctionArgList Args;
1768 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1769 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1770 ImplicitParamDecl::Other);
1771 Args.push_back(&Dst);
1772
1773 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1774 CGM.getContext().VoidPtrTy, Args);
1775 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1776 std::string Name = getName({"__kmpc_global_ctor_", ""});
1777 llvm::Function *Fn =
1778 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1779 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1780 Args, Loc, Loc);
1781 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1782 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1783 CGM.getContext().VoidPtrTy, Dst.getLocation());
1784 Address Arg = Address(ArgVal, VDAddr.getAlignment());
1785 Arg = CtorCGF.Builder.CreateElementBitCast(
1786 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
1787 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1788 /*IsInitializer=*/true);
1789 ArgVal = CtorCGF.EmitLoadOfScalar(
1790 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1791 CGM.getContext().VoidPtrTy, Dst.getLocation());
1792 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1793 CtorCGF.FinishFunction();
1794 Ctor = Fn;
1795 }
1796 if (VD->getType().isDestructedType() != QualType::DK_none) {
1797 // Generate function that emits destructor call for the threadprivate copy
1798 // of the variable VD
1799 CodeGenFunction DtorCGF(CGM);
1800 FunctionArgList Args;
1801 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1802 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1803 ImplicitParamDecl::Other);
1804 Args.push_back(&Dst);
1805
1806 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1807 CGM.getContext().VoidTy, Args);
1808 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1809 std::string Name = getName({"__kmpc_global_dtor_", ""});
1810 llvm::Function *Fn =
1811 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1812 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1813 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1814 Loc, Loc);
1815 // Create a scope with an artificial location for the body of this function.
1816 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1817 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1818 DtorCGF.GetAddrOfLocalVar(&Dst),
1819 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
1820 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
1821 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1822 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1823 DtorCGF.FinishFunction();
1824 Dtor = Fn;
1825 }
1826 // Do not emit init function if it is not required.
1827 if (!Ctor && !Dtor)
1828 return nullptr;
1829
1830 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1831 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1832 /*isVarArg=*/false)
1833 ->getPointerTo();
1834 // Copying constructor for the threadprivate variable.
1835 // Must be NULL - reserved by runtime, but currently it requires that this
1836 // parameter is always NULL. Otherwise it fires assertion.
1837 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1838 if (Ctor == nullptr) {
1839 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1840 /*isVarArg=*/false)
1841 ->getPointerTo();
1842 Ctor = llvm::Constant::getNullValue(CtorTy);
1843 }
1844 if (Dtor == nullptr) {
1845 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1846 /*isVarArg=*/false)
1847 ->getPointerTo();
1848 Dtor = llvm::Constant::getNullValue(DtorTy);
1849 }
1850 if (!CGF) {
1851 auto *InitFunctionTy =
1852 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1853 std::string Name = getName({"__omp_threadprivate_init_", ""});
1854 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1855 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
1856 CodeGenFunction InitCGF(CGM);
1857 FunctionArgList ArgList;
1858 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1859 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1860 Loc, Loc);
1861 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1862 InitCGF.FinishFunction();
1863 return InitFunction;
1864 }
1865 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1866 }
1867 return nullptr;
1868}
1869
1870bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
1871 llvm::GlobalVariable *Addr,
1872 bool PerformInit) {
1873 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
1874 !CGM.getLangOpts().OpenMPIsDevice)
1875 return false;
1876 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1877 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1878 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1879 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1880 HasRequiresUnifiedSharedMemory))
1881 return CGM.getLangOpts().OpenMPIsDevice;
1882 VD = VD->getDefinition(CGM.getContext());
1883 assert(VD && "Unknown VarDecl")((VD && "Unknown VarDecl") ? static_cast<void> (
0) : __assert_fail ("VD && \"Unknown VarDecl\"", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1883, __PRETTY_FUNCTION__))
;
1884
1885 if (!DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
1886 return CGM.getLangOpts().OpenMPIsDevice;
1887
1888 QualType ASTTy = VD->getType();
1889 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
1890
1891 // Produce the unique prefix to identify the new target regions. We use
1892 // the source location of the variable declaration which we know to not
1893 // conflict with any target region.
1894 unsigned DeviceID;
1895 unsigned FileID;
1896 unsigned Line;
1897 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
1898 SmallString<128> Buffer, Out;
1899 {
1900 llvm::raw_svector_ostream OS(Buffer);
1901 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
1902 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
1903 }
1904
1905 const Expr *Init = VD->getAnyInitializer();
1906 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1907 llvm::Constant *Ctor;
1908 llvm::Constant *ID;
1909 if (CGM.getLangOpts().OpenMPIsDevice) {
1910 // Generate function that re-emits the declaration's initializer into
1911 // the threadprivate copy of the variable VD
1912 CodeGenFunction CtorCGF(CGM);
1913
1914 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1915 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1916 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1917 FTy, Twine(Buffer, "_ctor"), FI, Loc);
1918 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
1919 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1920 FunctionArgList(), Loc, Loc);
1921 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
1922 CtorCGF.EmitAnyExprToMem(Init,
1923 Address(Addr, CGM.getContext().getDeclAlign(VD)),
1924 Init->getType().getQualifiers(),
1925 /*IsInitializer=*/true);
1926 CtorCGF.FinishFunction();
1927 Ctor = Fn;
1928 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1929 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
1930 } else {
1931 Ctor = new llvm::GlobalVariable(
1932 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1933 llvm::GlobalValue::PrivateLinkage,
1934 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
1935 ID = Ctor;
1936 }
1937
1938 // Register the information for the entry associated with the constructor.
1939 Out.clear();
1940 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1941 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
1942 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
1943 }
1944 if (VD->getType().isDestructedType() != QualType::DK_none) {
1945 llvm::Constant *Dtor;
1946 llvm::Constant *ID;
1947 if (CGM.getLangOpts().OpenMPIsDevice) {
1948 // Generate function that emits destructor call for the threadprivate
1949 // copy of the variable VD
1950 CodeGenFunction DtorCGF(CGM);
1951
1952 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1953 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1954 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1955 FTy, Twine(Buffer, "_dtor"), FI, Loc);
1956 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1957 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1958 FunctionArgList(), Loc, Loc);
1959 // Create a scope with an artificial location for the body of this
1960 // function.
1961 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1962 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
1963 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1964 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1965 DtorCGF.FinishFunction();
1966 Dtor = Fn;
1967 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1968 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
1969 } else {
1970 Dtor = new llvm::GlobalVariable(
1971 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1972 llvm::GlobalValue::PrivateLinkage,
1973 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
1974 ID = Dtor;
1975 }
1976 // Register the information for the entry associated with the destructor.
1977 Out.clear();
1978 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1979 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
1980 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
1981 }
1982 return CGM.getLangOpts().OpenMPIsDevice;
1983}
1984
1985Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1986 QualType VarType,
1987 StringRef Name) {
1988 std::string Suffix = getName({"artificial", ""});
1989 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1990 llvm::Value *GAddr =
1991 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
1992 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1993 CGM.getTarget().isTLSSupported()) {
1994 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
1995 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
1996 }
1997 std::string CacheSuffix = getName({"cache", ""});
1998 llvm::Value *Args[] = {
1999 emitUpdateLocation(CGF, SourceLocation()),
2000 getThreadID(CGF, SourceLocation()),
2001 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2002 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2003 /*isSigned=*/false),
2004 getOrCreateInternalVariable(
2005 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2006 return Address(
2007 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2008 CGF.EmitRuntimeCall(
2009 OMPBuilder.getOrCreateRuntimeFunction(
2010 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
2011 Args),
2012 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2013 CGM.getContext().getTypeAlignInChars(VarType));
2014}
2015
2016void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
2017 const RegionCodeGenTy &ThenGen,
2018 const RegionCodeGenTy &ElseGen) {
2019 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2020
2021 // If the condition constant folds and can be elided, try to avoid emitting
2022 // the condition and the dead arm of the if/else.
2023 bool CondConstant;
2024 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
9
Assuming the condition is true
10
Taking true branch
2025 if (CondConstant)
11
Assuming 'CondConstant' is true
12
Taking true branch
2026 ThenGen(CGF);
13
Calling 'RegionCodeGenTy::operator()'
2027 else
2028 ElseGen(CGF);
2029 return;
2030 }
2031
2032 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2033 // emit the conditional branch.
2034 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2035 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2036 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2037 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2038
2039 // Emit the 'then' code.
2040 CGF.EmitBlock(ThenBlock);
2041 ThenGen(CGF);
2042 CGF.EmitBranch(ContBlock);
2043 // Emit the 'else' code if present.
2044 // There is no need to emit line number for unconditional branch.
2045 (void)ApplyDebugLocation::CreateEmpty(CGF);
2046 CGF.EmitBlock(ElseBlock);
2047 ElseGen(CGF);
2048 // There is no need to emit line number for unconditional branch.
2049 (void)ApplyDebugLocation::CreateEmpty(CGF);
2050 CGF.EmitBranch(ContBlock);
2051 // Emit the continuation block for code after the if.
2052 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2053}
2054
2055void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2056 llvm::Function *OutlinedFn,
2057 ArrayRef<llvm::Value *> CapturedVars,
2058 const Expr *IfCond) {
2059 if (!CGF.HaveInsertPoint())
2060 return;
2061 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2062 auto &M = CGM.getModule();
2063 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2064 this](CodeGenFunction &CGF, PrePostActionTy &) {
2065 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2066 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2067 llvm::Value *Args[] = {
2068 RTLoc,
2069 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2070 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2071 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2072 RealArgs.append(std::begin(Args), std::end(Args));
2073 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2074
2075 llvm::FunctionCallee RTLFn =
2076 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2077 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2078 };
2079 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2080 this](CodeGenFunction &CGF, PrePostActionTy &) {
2081 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2082 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2083 // Build calls:
2084 // __kmpc_serialized_parallel(&Loc, GTid);
2085 llvm::Value *Args[] = {RTLoc, ThreadID};
2086 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2087 M, OMPRTL___kmpc_serialized_parallel),
2088 Args);
2089
2090 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2091 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2092 Address ZeroAddrBound =
2093 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2094 /*Name=*/".bound.zero.addr");
2095 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
2096 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2097 // ThreadId for serialized parallels is 0.
2098 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2099 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
2100 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2101 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2102
2103 // __kmpc_end_serialized_parallel(&Loc, GTid);
2104 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2105 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2106 M, OMPRTL___kmpc_end_serialized_parallel),
2107 EndArgs);
2108 };
2109 if (IfCond) {
2110 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2111 } else {
2112 RegionCodeGenTy ThenRCG(ThenGen);
2113 ThenRCG(CGF);
2114 }
2115}
2116
2117// If we're inside an (outlined) parallel region, use the region info's
2118// thread-ID variable (it is passed in a first argument of the outlined function
2119// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2120// regular serial code region, get thread ID by calling kmp_int32
2121// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2122// return the address of that temp.
2123Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2124 SourceLocation Loc) {
2125 if (auto *OMPRegionInfo =
2126 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2127 if (OMPRegionInfo->getThreadIDVariable())
2128 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2129
2130 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2131 QualType Int32Ty =
2132 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2133 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2134 CGF.EmitStoreOfScalar(ThreadID,
2135 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2136
2137 return ThreadIDTemp;
2138}
2139
2140llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
2141 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2142 SmallString<256> Buffer;
2143 llvm::raw_svector_ostream Out(Buffer);
2144 Out << Name;
2145 StringRef RuntimeName = Out.str();
2146 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2147 if (Elem.second) {
2148 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2149, __PRETTY_FUNCTION__))
2149 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2149, __PRETTY_FUNCTION__))
;
2150 return &*Elem.second;
2151 }
2152
2153 return Elem.second = new llvm::GlobalVariable(
2154 CGM.getModule(), Ty, /*IsConstant*/ false,
2155 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2156 Elem.first(), /*InsertBefore=*/nullptr,
2157 llvm::GlobalValue::NotThreadLocal, AddressSpace);
2158}
2159
2160llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2161 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2162 std::string Name = getName({Prefix, "var"});
2163 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2164}
2165
2166namespace {
2167/// Common pre(post)-action for different OpenMP constructs.
2168class CommonActionTy final : public PrePostActionTy {
2169 llvm::FunctionCallee EnterCallee;
2170 ArrayRef<llvm::Value *> EnterArgs;
2171 llvm::FunctionCallee ExitCallee;
2172 ArrayRef<llvm::Value *> ExitArgs;
2173 bool Conditional;
2174 llvm::BasicBlock *ContBlock = nullptr;
2175
2176public:
2177 CommonActionTy(llvm::FunctionCallee EnterCallee,
2178 ArrayRef<llvm::Value *> EnterArgs,
2179 llvm::FunctionCallee ExitCallee,
2180 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2181 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2182 ExitArgs(ExitArgs), Conditional(Conditional) {}
2183 void Enter(CodeGenFunction &CGF) override {
2184 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2185 if (Conditional) {
2186 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2187 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2188 ContBlock = CGF.createBasicBlock("omp_if.end");
2189 // Generate the branch (If-stmt)
2190 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2191 CGF.EmitBlock(ThenBlock);
2192 }
2193 }
2194 void Done(CodeGenFunction &CGF) {
2195 // Emit the rest of blocks/branches
2196 CGF.EmitBranch(ContBlock);
2197 CGF.EmitBlock(ContBlock, true);
2198 }
2199 void Exit(CodeGenFunction &CGF) override {
2200 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2201 }
2202};
2203} // anonymous namespace
2204
2205void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2206 StringRef CriticalName,
2207 const RegionCodeGenTy &CriticalOpGen,
2208 SourceLocation Loc, const Expr *Hint) {
2209 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2210 // CriticalOpGen();
2211 // __kmpc_end_critical(ident_t *, gtid, Lock);
2212 // Prepare arguments and build a call to __kmpc_critical
2213 if (!CGF.HaveInsertPoint())
2214 return;
2215 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2216 getCriticalRegionLock(CriticalName)};
2217 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2218 std::end(Args));
2219 if (Hint) {
2220 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2221 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2222 }
2223 CommonActionTy Action(
2224 OMPBuilder.getOrCreateRuntimeFunction(
2225 CGM.getModule(),
2226 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2227 EnterArgs,
2228 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2229 OMPRTL___kmpc_end_critical),
2230 Args);
2231 CriticalOpGen.setAction(Action);
2232 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2233}
2234
2235void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2236 const RegionCodeGenTy &MasterOpGen,
2237 SourceLocation Loc) {
2238 if (!CGF.HaveInsertPoint())
2239 return;
2240 // if(__kmpc_master(ident_t *, gtid)) {
2241 // MasterOpGen();
2242 // __kmpc_end_master(ident_t *, gtid);
2243 // }
2244 // Prepare arguments and build a call to __kmpc_master
2245 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2246 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2247 CGM.getModule(), OMPRTL___kmpc_master),
2248 Args,
2249 OMPBuilder.getOrCreateRuntimeFunction(
2250 CGM.getModule(), OMPRTL___kmpc_end_master),
2251 Args,
2252 /*Conditional=*/true);
2253 MasterOpGen.setAction(Action);
2254 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2255 Action.Done(CGF);
2256}
2257
2258void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2259 SourceLocation Loc) {
2260 if (!CGF.HaveInsertPoint())
2261 return;
2262 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2263 OMPBuilder.CreateTaskyield(CGF.Builder);
2264 } else {
2265 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2266 llvm::Value *Args[] = {
2267 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2268 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
2269 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2270 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2271 Args);
2272 }
2273
2274 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2275 Region->emitUntiedSwitch(CGF);
2276}
2277
2278void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2279 const RegionCodeGenTy &TaskgroupOpGen,
2280 SourceLocation Loc) {
2281 if (!CGF.HaveInsertPoint())
2282 return;
2283 // __kmpc_taskgroup(ident_t *, gtid);
2284 // TaskgroupOpGen();
2285 // __kmpc_end_taskgroup(ident_t *, gtid);
2286 // Prepare arguments and build a call to __kmpc_taskgroup
2287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2288 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2289 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2290 Args,
2291 OMPBuilder.getOrCreateRuntimeFunction(
2292 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2293 Args);
2294 TaskgroupOpGen.setAction(Action);
2295 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2296}
2297
2298/// Given an array of pointers to variables, project the address of a
2299/// given variable.
2300static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2301 unsigned Index, const VarDecl *Var) {
2302 // Pull out the pointer to the variable.
2303 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2304 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2305
2306 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
2307 Addr = CGF.Builder.CreateElementBitCast(
2308 Addr, CGF.ConvertTypeForMem(Var->getType()));
2309 return Addr;
2310}
2311
2312static llvm::Value *emitCopyprivateCopyFunction(
2313 CodeGenModule &CGM, llvm::Type *ArgsType,
2314 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2315 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2316 SourceLocation Loc) {
2317 ASTContext &C = CGM.getContext();
2318 // void copy_func(void *LHSArg, void *RHSArg);
2319 FunctionArgList Args;
2320 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2321 ImplicitParamDecl::Other);
2322 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2323 ImplicitParamDecl::Other);
2324 Args.push_back(&LHSArg);
2325 Args.push_back(&RHSArg);
2326 const auto &CGFI =
2327 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
2328 std::string Name =
2329 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
2330 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
2331 llvm::GlobalValue::InternalLinkage, Name,
2332 &CGM.getModule());
2333 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
2334 Fn->setDoesNotRecurse();
2335 CodeGenFunction CGF(CGM);
2336 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2337 // Dest = (void*[n])(LHSArg);
2338 // Src = (void*[n])(RHSArg);
2339 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2340 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2341 ArgsType), CGF.getPointerAlign());
2342 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2343 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2344 ArgsType), CGF.getPointerAlign());
2345 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2346 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2347 // ...
2348 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2349 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2350 const auto *DestVar =
2351 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2352 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2353
2354 const auto *SrcVar =
2355 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2356 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2357
2358 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2359 QualType Type = VD->getType();
2360 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2361 }
2362 CGF.FinishFunction();
2363 return Fn;
2364}
2365
2366void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2367 const RegionCodeGenTy &SingleOpGen,
2368 SourceLocation Loc,
2369 ArrayRef<const Expr *> CopyprivateVars,
2370 ArrayRef<const Expr *> SrcExprs,
2371 ArrayRef<const Expr *> DstExprs,
2372 ArrayRef<const Expr *> AssignmentOps) {
2373 if (!CGF.HaveInsertPoint())
2374 return;
2375 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
2376 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
2377 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
;
2378 ASTContext &C = CGM.getContext();
2379 // int32 did_it = 0;
2380 // if(__kmpc_single(ident_t *, gtid)) {
2381 // SingleOpGen();
2382 // __kmpc_end_single(ident_t *, gtid);
2383 // did_it = 1;
2384 // }
2385 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2386 // <copy_func>, did_it);
2387
2388 Address DidIt = Address::invalid();
2389 if (!CopyprivateVars.empty()) {
2390 // int32 did_it = 0;
2391 QualType KmpInt32Ty =
2392 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2393 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2394 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2395 }
2396 // Prepare arguments and build a call to __kmpc_single
2397 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2398 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2399 CGM.getModule(), OMPRTL___kmpc_single),
2400 Args,
2401 OMPBuilder.getOrCreateRuntimeFunction(
2402 CGM.getModule(), OMPRTL___kmpc_end_single),
2403 Args,
2404 /*Conditional=*/true);
2405 SingleOpGen.setAction(Action);
2406 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2407 if (DidIt.isValid()) {
2408 // did_it = 1;
2409 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2410 }
2411 Action.Done(CGF);
2412 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2413 // <copy_func>, did_it);
2414 if (DidIt.isValid()) {
2415 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2416 QualType CopyprivateArrayTy = C.getConstantArrayType(
2417 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
2418 /*IndexTypeQuals=*/0);
2419 // Create a list of all private variables for copyprivate.
2420 Address CopyprivateList =
2421 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2422 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2423 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2424 CGF.Builder.CreateStore(
2425 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2426 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2427 CGF.VoidPtrTy),
2428 Elem);
2429 }
2430 // Build function that copies private values from single region to all other
2431 // threads in the corresponding parallel region.
2432 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2433 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
2434 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
2435 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2436 Address CL =
2437 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2438 CGF.VoidPtrTy);
2439 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2440 llvm::Value *Args[] = {
2441 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2442 getThreadID(CGF, Loc), // i32 <gtid>
2443 BufSize, // size_t <buf_size>
2444 CL.getPointer(), // void *<copyprivate list>
2445 CpyFn, // void (*) (void *, void *) <copy_func>
2446 DidItVal // i32 did_it
2447 };
2448 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2449 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2450 Args);
2451 }
2452}
2453
2454void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2455 const RegionCodeGenTy &OrderedOpGen,
2456 SourceLocation Loc, bool IsThreads) {
2457 if (!CGF.HaveInsertPoint())
2458 return;
2459 // __kmpc_ordered(ident_t *, gtid);
2460 // OrderedOpGen();
2461 // __kmpc_end_ordered(ident_t *, gtid);
2462 // Prepare arguments and build a call to __kmpc_ordered
2463 if (IsThreads) {
2464 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2465 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2466 CGM.getModule(), OMPRTL___kmpc_ordered),
2467 Args,
2468 OMPBuilder.getOrCreateRuntimeFunction(
2469 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2470 Args);
2471 OrderedOpGen.setAction(Action);
2472 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2473 return;
2474 }
2475 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2476}
2477
2478unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2479 unsigned Flags;
2480 if (Kind == OMPD_for)
2481 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2482 else if (Kind == OMPD_sections)
2483 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2484 else if (Kind == OMPD_single)
2485 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2486 else if (Kind == OMPD_barrier)
2487 Flags = OMP_IDENT_BARRIER_EXPL;
2488 else
2489 Flags = OMP_IDENT_BARRIER_IMPL;
2490 return Flags;
2491}
2492
2493void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2494 CodeGenFunction &CGF, const OMPLoopDirective &S,
2495 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2496 // Check if the loop directive is actually a doacross loop directive. In this
2497 // case choose static, 1 schedule.
2498 if (llvm::any_of(
2499 S.getClausesOfKind<OMPOrderedClause>(),
2500 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2501 ScheduleKind = OMPC_SCHEDULE_static;
2502 // Chunk size is 1 in this case.
2503 llvm::APInt ChunkSize(32, 1);
2504 ChunkExpr = IntegerLiteral::Create(
2505 CGF.getContext(), ChunkSize,
2506 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2507 SourceLocation());
2508 }
2509}
2510
2511void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2512 OpenMPDirectiveKind Kind, bool EmitChecks,
2513 bool ForceSimpleCall) {
2514 // Check if we should use the OMPBuilder
2515 auto *OMPRegionInfo =
2516 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2517 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2518 CGF.Builder.restoreIP(OMPBuilder.CreateBarrier(
2519 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
2520 return;
2521 }
2522
2523 if (!CGF.HaveInsertPoint())
2524 return;
2525 // Build call __kmpc_cancel_barrier(loc, thread_id);
2526 // Build call __kmpc_barrier(loc, thread_id);
2527 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2528 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2529 // thread_id);
2530 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2531 getThreadID(CGF, Loc)};
2532 if (OMPRegionInfo) {
2533 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2534 llvm::Value *Result = CGF.EmitRuntimeCall(
2535 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2536 OMPRTL___kmpc_cancel_barrier),
2537 Args);
2538 if (EmitChecks) {
2539 // if (__kmpc_cancel_barrier()) {
2540 // exit from construct;
2541 // }
2542 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2543 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2544 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2545 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2546 CGF.EmitBlock(ExitBB);
2547 // exit from construct;
2548 CodeGenFunction::JumpDest CancelDestination =
2549 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2550 CGF.EmitBranchThroughCleanup(CancelDestination);
2551 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2552 }
2553 return;
2554 }
2555 }
2556 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2557 CGM.getModule(), OMPRTL___kmpc_barrier),
2558 Args);
2559}
2560
2561/// Map the OpenMP loop schedule to the runtime enumeration.
2562static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2563 bool Chunked, bool Ordered) {
2564 switch (ScheduleKind) {
2565 case OMPC_SCHEDULE_static:
2566 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2567 : (Ordered ? OMP_ord_static : OMP_sch_static);
2568 case OMPC_SCHEDULE_dynamic:
2569 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2570 case OMPC_SCHEDULE_guided:
2571 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2572 case OMPC_SCHEDULE_runtime:
2573 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2574 case OMPC_SCHEDULE_auto:
2575 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2576 case OMPC_SCHEDULE_unknown:
2577 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2577, __PRETTY_FUNCTION__))
;
2578 return Ordered ? OMP_ord_static : OMP_sch_static;
2579 }
2580 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2580)
;
2581}
2582
2583/// Map the OpenMP distribute schedule to the runtime enumeration.
2584static OpenMPSchedType
2585getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2586 // only static is allowed for dist_schedule
2587 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2588}
2589
2590bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2591 bool Chunked) const {
2592 OpenMPSchedType Schedule =
2593 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2594 return Schedule == OMP_sch_static;
2595}
2596
2597bool CGOpenMPRuntime::isStaticNonchunked(
2598 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2599 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2600 return Schedule == OMP_dist_sch_static;
2601}
2602
2603bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2604 bool Chunked) const {
2605 OpenMPSchedType Schedule =
2606 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2607 return Schedule == OMP_sch_static_chunked;
2608}
2609
2610bool CGOpenMPRuntime::isStaticChunked(
2611 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2612 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2613 return Schedule == OMP_dist_sch_static_chunked;
2614}
2615
2616bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2617 OpenMPSchedType Schedule =
2618 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2619 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2619, __PRETTY_FUNCTION__))
;
2620 return Schedule != OMP_sch_static;
2621}
2622
2623static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2624 OpenMPScheduleClauseModifier M1,
2625 OpenMPScheduleClauseModifier M2) {
2626 int Modifier = 0;
2627 switch (M1) {
2628 case OMPC_SCHEDULE_MODIFIER_monotonic:
2629 Modifier = OMP_sch_modifier_monotonic;
2630 break;
2631 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2632 Modifier = OMP_sch_modifier_nonmonotonic;
2633 break;
2634 case OMPC_SCHEDULE_MODIFIER_simd:
2635 if (Schedule == OMP_sch_static_chunked)
2636 Schedule = OMP_sch_static_balanced_chunked;
2637 break;
2638 case OMPC_SCHEDULE_MODIFIER_last:
2639 case OMPC_SCHEDULE_MODIFIER_unknown:
2640 break;
2641 }
2642 switch (M2) {
2643 case OMPC_SCHEDULE_MODIFIER_monotonic:
2644 Modifier = OMP_sch_modifier_monotonic;
2645 break;
2646 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2647 Modifier = OMP_sch_modifier_nonmonotonic;
2648 break;
2649 case OMPC_SCHEDULE_MODIFIER_simd:
2650 if (Schedule == OMP_sch_static_chunked)
2651 Schedule = OMP_sch_static_balanced_chunked;
2652 break;
2653 case OMPC_SCHEDULE_MODIFIER_last:
2654 case OMPC_SCHEDULE_MODIFIER_unknown:
2655 break;
2656 }
2657 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2658 // If the static schedule kind is specified or if the ordered clause is
2659 // specified, and if the nonmonotonic modifier is not specified, the effect is
2660 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2661 // modifier is specified, the effect is as if the nonmonotonic modifier is
2662 // specified.
2663 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2664 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2665 Schedule == OMP_sch_static_balanced_chunked ||
2666 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2667 Schedule == OMP_dist_sch_static_chunked ||
2668 Schedule == OMP_dist_sch_static))
2669 Modifier = OMP_sch_modifier_nonmonotonic;
2670 }
2671 return Schedule | Modifier;
2672}
2673
2674void CGOpenMPRuntime::emitForDispatchInit(
2675 CodeGenFunction &CGF, SourceLocation Loc,
2676 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2677 bool Ordered, const DispatchRTInput &DispatchValues) {
2678 if (!CGF.HaveInsertPoint())
2679 return;
2680 OpenMPSchedType Schedule = getRuntimeSchedule(
2681 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
2682 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2683 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2684 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2685 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
;
2686 // Call __kmpc_dispatch_init(
2687 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2688 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2689 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2690
2691 // If the Chunk was not specified in the clause - use default value 1.
2692 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2693 : CGF.Builder.getIntN(IVSize, 1);
2694 llvm::Value *Args[] = {
2695 emitUpdateLocation(CGF, Loc),
2696 getThreadID(CGF, Loc),
2697 CGF.Builder.getInt32(addMonoNonMonoModifier(
2698 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
2699 DispatchValues.LB, // Lower
2700 DispatchValues.UB, // Upper
2701 CGF.Builder.getIntN(IVSize, 1), // Stride
2702 Chunk // Chunk
2703 };
2704 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2705}
2706
2707static void emitForStaticInitCall(
2708 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2709 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2710 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2711 const CGOpenMPRuntime::StaticRTInput &Values) {
2712 if (!CGF.HaveInsertPoint())
2713 return;
2714
2715 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2715, __PRETTY_FUNCTION__))
;
2716 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2717 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2718 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2719 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2720 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
;
2721
2722 // Call __kmpc_for_static_init(
2723 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2724 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2725 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2726 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2727 llvm::Value *Chunk = Values.Chunk;
2728 if (Chunk == nullptr) {
2729 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
2730 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
2731 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
;
2732 // If the Chunk was not specified in the clause - use default value 1.
2733 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2734 } else {
2735 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2736 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2737 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2738 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2739 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
;
2740 }
2741 llvm::Value *Args[] = {
2742 UpdateLocation,
2743 ThreadId,
2744 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2745 M2)), // Schedule type
2746 Values.IL.getPointer(), // &isLastIter
2747 Values.LB.getPointer(), // &LB
2748 Values.UB.getPointer(), // &UB
2749 Values.ST.getPointer(), // &Stride
2750 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2751 Chunk // Chunk
2752 };
2753 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2754}
2755
2756void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2757 SourceLocation Loc,
2758 OpenMPDirectiveKind DKind,
2759 const OpenMPScheduleTy &ScheduleKind,
2760 const StaticRTInput &Values) {
2761 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
2762 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
2763 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2764, __PRETTY_FUNCTION__))
2764 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2764, __PRETTY_FUNCTION__))
;
2765 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2766 isOpenMPLoopDirective(DKind)
2767 ? OMP_IDENT_WORK_LOOP
2768 : OMP_IDENT_WORK_SECTIONS);
2769 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2770 llvm::FunctionCallee StaticInitFunction =
2771 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2772 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2773 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2774 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
2775}
2776
2777void CGOpenMPRuntime::emitDistributeStaticInit(
2778 CodeGenFunction &CGF, SourceLocation Loc,
2779 OpenMPDistScheduleClauseKind SchedKind,
2780 const CGOpenMPRuntime::StaticRTInput &Values) {
2781 OpenMPSchedType ScheduleNum =
2782 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
2783 llvm::Value *UpdatedLocation =
2784 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2785 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2786 llvm::FunctionCallee StaticInitFunction =
2787 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2788 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2789 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
2790 OMPC_SCHEDULE_MODIFIER_unknown, Values);
2791}
2792
2793void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2794 SourceLocation Loc,
2795 OpenMPDirectiveKind DKind) {
2796 if (!CGF.HaveInsertPoint())
2797 return;
2798 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2799 llvm::Value *Args[] = {
2800 emitUpdateLocation(CGF, Loc,
2801 isOpenMPDistributeDirective(DKind)
2802 ? OMP_IDENT_WORK_DISTRIBUTE
2803 : isOpenMPLoopDirective(DKind)
2804 ? OMP_IDENT_WORK_LOOP
2805 : OMP_IDENT_WORK_SECTIONS),
2806 getThreadID(CGF, Loc)};
2807 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2808 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2809 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2810 Args);
2811}
2812
2813void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2814 SourceLocation Loc,
2815 unsigned IVSize,
2816 bool IVSigned) {
2817 if (!CGF.HaveInsertPoint())
2818 return;
2819 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2820 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2821 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2822}
2823
2824llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2825 SourceLocation Loc, unsigned IVSize,
2826 bool IVSigned, Address IL,
2827 Address LB, Address UB,
2828 Address ST) {
2829 // Call __kmpc_dispatch_next(
2830 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2831 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2832 // kmp_int[32|64] *p_stride);
2833 llvm::Value *Args[] = {
2834 emitUpdateLocation(CGF, Loc),
2835 getThreadID(CGF, Loc),
2836 IL.getPointer(), // &isLastIter
2837 LB.getPointer(), // &Lower
2838 UB.getPointer(), // &Upper
2839 ST.getPointer() // &Stride
2840 };
2841 llvm::Value *Call =
2842 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2843 return CGF.EmitScalarConversion(
2844 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2845 CGF.getContext().BoolTy, Loc);
2846}
2847
2848void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2849 llvm::Value *NumThreads,
2850 SourceLocation Loc) {
2851 if (!CGF.HaveInsertPoint())
2852 return;
2853 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2854 llvm::Value *Args[] = {
2855 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2856 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2857 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2858 CGM.getModule(), OMPRTL___kmpc_push_num_threads),
2859 Args);
2860}
2861
2862void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2863 ProcBindKind ProcBind,
2864 SourceLocation Loc) {
2865 if (!CGF.HaveInsertPoint())
2866 return;
2867 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2867, __PRETTY_FUNCTION__))
;
2868 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2869 llvm::Value *Args[] = {
2870 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2871 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
2872 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2873 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2874 Args);
2875}
2876
2877void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2878 SourceLocation Loc, llvm::AtomicOrdering AO) {
2879 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2880 OMPBuilder.CreateFlush(CGF.Builder);
2881 } else {
2882 if (!CGF.HaveInsertPoint())
2883 return;
2884 // Build call void __kmpc_flush(ident_t *loc)
2885 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2886 CGM.getModule(), OMPRTL___kmpc_flush),
2887 emitUpdateLocation(CGF, Loc));
2888 }
2889}
2890
2891namespace {
2892/// Indexes of fields for type kmp_task_t.
2893enum KmpTaskTFields {
2894 /// List of shared variables.
2895 KmpTaskTShareds,
2896 /// Task routine.
2897 KmpTaskTRoutine,
2898 /// Partition id for the untied tasks.
2899 KmpTaskTPartId,
2900 /// Function with call of destructors for private variables.
2901 Data1,
2902 /// Task priority.
2903 Data2,
2904 /// (Taskloops only) Lower bound.
2905 KmpTaskTLowerBound,
2906 /// (Taskloops only) Upper bound.
2907 KmpTaskTUpperBound,
2908 /// (Taskloops only) Stride.
2909 KmpTaskTStride,
2910 /// (Taskloops only) Is last iteration flag.
2911 KmpTaskTLastIter,
2912 /// (Taskloops only) Reduction data.
2913 KmpTaskTReductions,
2914};
2915} // anonymous namespace
2916
2917bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
2918 return OffloadEntriesTargetRegion.empty() &&
2919 OffloadEntriesDeviceGlobalVar.empty();
2920}
2921
2922/// Initialize target region entry.
2923void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2924 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2925 StringRef ParentName, unsigned LineNum,
2926 unsigned Order) {
2927 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
2928 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
2929 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
;
2930 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
2931 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
2932 OMPTargetRegionEntryTargetRegion);
2933 ++OffloadingEntriesNum;
2934}
2935
2936void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2937 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2938 StringRef ParentName, unsigned LineNum,
2939 llvm::Constant *Addr, llvm::Constant *ID,
2940 OMPTargetRegionEntryKind Flags) {
2941 // If we are emitting code for a target, the entry is already initialized,
2942 // only has to be registered.
2943 if (CGM.getLangOpts().OpenMPIsDevice) {
2944 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
2945 unsigned DiagID = CGM.getDiags().getCustomDiagID(
2946 DiagnosticsEngine::Error,
2947 "Unable to find target region on line '%0' in the device code.");
2948 CGM.getDiags().Report(DiagID) << LineNum;
2949 return;
2950 }
2951 auto &Entry =
2952 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
2953 assert(Entry.isValid() && "Entry not initialized!")((Entry.isValid() && "Entry not initialized!") ? static_cast
<void> (0) : __assert_fail ("Entry.isValid() && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2953, __PRETTY_FUNCTION__))
;
2954 Entry.setAddress(Addr);
2955 Entry.setID(ID);
2956 Entry.setFlags(Flags);
2957 } else {
2958 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
2959 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
2960 ++OffloadingEntriesNum;
2961 }
2962}
2963
2964bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2965 unsigned DeviceID, unsigned FileID, StringRef ParentName,
2966 unsigned LineNum) const {
2967 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2968 if (PerDevice == OffloadEntriesTargetRegion.end())
2969 return false;
2970 auto PerFile = PerDevice->second.find(FileID);
2971 if (PerFile == PerDevice->second.end())
2972 return false;
2973 auto PerParentName = PerFile->second.find(ParentName);
2974 if (PerParentName == PerFile->second.end())
2975 return false;
2976 auto PerLine = PerParentName->second.find(LineNum);
2977 if (PerLine == PerParentName->second.end())
2978 return false;
2979 // Fail if this entry is already registered.
2980 if (PerLine->second.getAddress() || PerLine->second.getID())
2981 return false;
2982 return true;
2983}
2984
2985void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
2986 const OffloadTargetRegionEntryInfoActTy &Action) {
2987 // Scan all target region entries and perform the provided action.
2988 for (const auto &D : OffloadEntriesTargetRegion)
2989 for (const auto &F : D.second)
2990 for (const auto &P : F.second)
2991 for (const auto &L : P.second)
2992 Action(D.first, F.first, P.first(), L.first, L.second);
2993}
2994
2995void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2996 initializeDeviceGlobalVarEntryInfo(StringRef Name,
2997 OMPTargetGlobalVarEntryKind Flags,
2998 unsigned Order) {
2999 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3001, __PRETTY_FUNCTION__))
3000 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3001, __PRETTY_FUNCTION__))
3001 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3001, __PRETTY_FUNCTION__))
;
3002 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3003 ++OffloadingEntriesNum;
3004}
3005
3006void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3007 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3008 CharUnits VarSize,
3009 OMPTargetGlobalVarEntryKind Flags,
3010 llvm::GlobalValue::LinkageTypes Linkage) {
3011 if (CGM.getLangOpts().OpenMPIsDevice) {
3012 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3013 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3014, __PRETTY_FUNCTION__))
3014 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3014, __PRETTY_FUNCTION__))
;
3015 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3016, __PRETTY_FUNCTION__))
3016 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3016, __PRETTY_FUNCTION__))
;
3017 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3018 if (Entry.getVarSize().isZero()) {
3019 Entry.setVarSize(VarSize);
3020 Entry.setLinkage(Linkage);
3021 }
3022 return;
3023 }
3024 Entry.setVarSize(VarSize);
3025 Entry.setLinkage(Linkage);
3026 Entry.setAddress(Addr);
3027 } else {
3028 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3029 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3030 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3031, __PRETTY_FUNCTION__))
3031 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3031, __PRETTY_FUNCTION__))
;
3032 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3033, __PRETTY_FUNCTION__))
3033 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3033, __PRETTY_FUNCTION__))
;
3034 if (Entry.getVarSize().isZero()) {
3035 Entry.setVarSize(VarSize);
3036 Entry.setLinkage(Linkage);
3037 }
3038 return;
3039 }
3040 OffloadEntriesDeviceGlobalVar.try_emplace(
3041 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3042 ++OffloadingEntriesNum;
3043 }
3044}
3045
3046void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3047 actOnDeviceGlobalVarEntriesInfo(
3048 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3049 // Scan all target region entries and perform the provided action.
3050 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3051 Action(E.getKey(), E.getValue());
3052}
3053
3054void CGOpenMPRuntime::createOffloadEntry(
3055 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3056 llvm::GlobalValue::LinkageTypes Linkage) {
3057 StringRef Name = Addr->getName();
3058 llvm::Module &M = CGM.getModule();
3059 llvm::LLVMContext &C = M.getContext();
3060
3061 // Create constant string with the name.
3062 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3063
3064 std::string StringName = getName({"omp_offloading", "entry_name"});
3065 auto *Str = new llvm::GlobalVariable(
3066 M, StrPtrInit->getType(), /*isConstant=*/true,
3067 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3068 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3069
3070 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
3071 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
3072 llvm::ConstantInt::get(CGM.SizeTy, Size),
3073 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3074 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3075 std::string EntryName = getName({"omp_offloading", "entry", ""});
3076 llvm::GlobalVariable *Entry = createGlobalStruct(
3077 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
3078 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
3079
3080 // The entry has to be created in the section the linker expects it to be.
3081 Entry->setSection("omp_offloading_entries");
3082}
3083
3084void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3085 // Emit the offloading entries and metadata so that the device codegen side
3086 // can easily figure out what to emit. The produced metadata looks like
3087 // this:
3088 //
3089 // !omp_offload.info = !{!1, ...}
3090 //
3091 // Right now we only generate metadata for function that contain target
3092 // regions.
3093
3094 // If we are in simd mode or there are no entries, we don't need to do
3095 // anything.
3096 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
3097 return;
3098
3099 llvm::Module &M = CGM.getModule();
3100 llvm::LLVMContext &C = M.getContext();
3101 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
3102 SourceLocation, StringRef>,
3103 16>
3104 OrderedEntries(OffloadEntriesInfoManager.size());
3105 llvm::SmallVector<StringRef, 16> ParentFunctions(
3106 OffloadEntriesInfoManager.size());
3107
3108 // Auxiliary methods to create metadata values and strings.
3109 auto &&GetMDInt = [this](unsigned V) {
3110 return llvm::ConstantAsMetadata::get(
3111 llvm::ConstantInt::get(CGM.Int32Ty, V));
3112 };
3113
3114 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3115
3116 // Create the offloading info metadata node.
3117 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3118
3119 // Create function that emits metadata for each target region entry;
3120 auto &&TargetRegionMetadataEmitter =
3121 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
3122 &GetMDString](
3123 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3124 unsigned Line,
3125 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3126 // Generate metadata for target regions. Each entry of this metadata
3127 // contains:
3128 // - Entry 0 -> Kind of this type of metadata (0).
3129 // - Entry 1 -> Device ID of the file where the entry was identified.
3130 // - Entry 2 -> File ID of the file where the entry was identified.
3131 // - Entry 3 -> Mangled name of the function where the entry was
3132 // identified.
3133 // - Entry 4 -> Line in the file where the entry was identified.
3134 // - Entry 5 -> Order the entry was created.
3135 // The first element of the metadata node is the kind.
3136 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3137 GetMDInt(FileID), GetMDString(ParentName),
3138 GetMDInt(Line), GetMDInt(E.getOrder())};
3139
3140 SourceLocation Loc;
3141 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3142 E = CGM.getContext().getSourceManager().fileinfo_end();
3143 I != E; ++I) {
3144 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
3145 I->getFirst()->getUniqueID().getFile() == FileID) {
3146 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3147 I->getFirst(), Line, 1);
3148 break;
3149 }
3150 }
3151 // Save this entry in the right position of the ordered entries array.
3152 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
3153 ParentFunctions[E.getOrder()] = ParentName;
3154
3155 // Add metadata to the named metadata node.
3156 MD->addOperand(llvm::MDNode::get(C, Ops));
3157 };
3158
3159 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3160 TargetRegionMetadataEmitter);
3161
3162 // Create function that emits metadata for each device global variable entry;
3163 auto &&DeviceGlobalVarMetadataEmitter =
3164 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3165 MD](StringRef MangledName,
3166 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3167 &E) {
3168 // Generate metadata for global variables. Each entry of this metadata
3169 // contains:
3170 // - Entry 0 -> Kind of this type of metadata (1).
3171 // - Entry 1 -> Mangled name of the variable.
3172 // - Entry 2 -> Declare target kind.
3173 // - Entry 3 -> Order the entry was created.
3174 // The first element of the metadata node is the kind.
3175 llvm::Metadata *Ops[] = {
3176 GetMDInt(E.getKind()), GetMDString(MangledName),
3177 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3178
3179 // Save this entry in the right position of the ordered entries array.
3180 OrderedEntries[E.getOrder()] =
3181 std::make_tuple(&E, SourceLocation(), MangledName);
3182
3183 // Add metadata to the named metadata node.
3184 MD->addOperand(llvm::MDNode::get(C, Ops));
3185 };
3186
3187 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3188 DeviceGlobalVarMetadataEmitter);
3189
3190 for (const auto &E : OrderedEntries) {
3191 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3191, __PRETTY_FUNCTION__))
;
3192 if (const auto *CE =
3193 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3194 std::get<0>(E))) {
3195 if (!CE->getID() || !CE->getAddress()) {
3196 // Do not blame the entry if the parent funtion is not emitted.
3197 StringRef FnName = ParentFunctions[CE->getOrder()];
3198 if (!CGM.GetGlobalValue(FnName))
3199 continue;
3200 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3201 DiagnosticsEngine::Error,
3202 "Offloading entry for target region in %0 is incorrect: either the "
3203 "address or the ID is invalid.");
3204 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
3205 continue;
3206 }
3207 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
3208 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3209 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
3210 OffloadEntryInfoDeviceGlobalVar>(
3211 std::get<0>(E))) {
3212 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3213 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3214 CE->getFlags());
3215 switch (Flags) {
3216 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3217 if (CGM.getLangOpts().OpenMPIsDevice &&
3218 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
3219 continue;
3220 if (!CE->getAddress()) {
3221 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3222 DiagnosticsEngine::Error, "Offloading entry for declare target "
3223 "variable %0 is incorrect: the "
3224 "address is invalid.");
3225 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
3226 continue;
3227 }
3228 // The vaiable has no definition - no need to add the entry.
3229 if (CE->getVarSize().isZero())
3230 continue;
3231 break;
3232 }
3233 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3234 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3236, __PRETTY_FUNCTION__))
3235 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3236, __PRETTY_FUNCTION__))
3236 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3236, __PRETTY_FUNCTION__))
;
3237 if (CGM.getLangOpts().OpenMPIsDevice)
3238 continue;
3239 if (!CE->getAddress()) {
3240 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3241 DiagnosticsEngine::Error,
3242 "Offloading entry for declare target variable is incorrect: the "
3243 "address is invalid.");
3244 CGM.getDiags().Report(DiagID);
3245 continue;
3246 }
3247 break;
3248 }
3249 createOffloadEntry(CE->getAddress(), CE->getAddress(),
3250 CE->getVarSize().getQuantity(), Flags,
3251 CE->getLinkage());
3252 } else {
3253 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3253)
;
3254 }
3255 }
3256}
3257
3258/// Loads all the offload entries information from the host IR
3259/// metadata.
3260void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3261 // If we are in target mode, load the metadata from the host IR. This code has
3262 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
3263
3264 if (!CGM.getLangOpts().OpenMPIsDevice)
3265 return;
3266
3267 if (CGM.getLangOpts().OMPHostIRFile.empty())
3268 return;
3269
3270 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3271 if (auto EC = Buf.getError()) {
3272 CGM.getDiags().Report(diag::err_cannot_open_file)
3273 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3274 return;
3275 }
3276
3277 llvm::LLVMContext C;
3278 auto ME = expectedToErrorOrAndEmitErrors(
3279 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3280
3281 if (auto EC = ME.getError()) {
3282 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3283 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
3284 CGM.getDiags().Report(DiagID)
3285 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3286 return;
3287 }
3288
3289 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
3290 if (!MD)
3291 return;
3292
3293 for (llvm::MDNode *MN : MD->operands()) {
3294 auto &&GetMDInt = [MN](unsigned Idx) {
3295 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3296 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3297 };
3298
3299 auto &&GetMDString = [MN](unsigned Idx) {
3300 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
3301 return V->getString();
3302 };
3303
3304 switch (GetMDInt(0)) {
3305 default:
3306 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3306)
;
3307 break;
3308 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3309 OffloadingEntryInfoTargetRegion:
3310 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3311 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
3312 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
3313 /*Order=*/GetMDInt(5));
3314 break;
3315 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3316 OffloadingEntryInfoDeviceGlobalVar:
3317 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
3318 /*MangledName=*/GetMDString(1),
3319 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3320 /*Flags=*/GetMDInt(2)),
3321 /*Order=*/GetMDInt(3));
3322 break;
3323 }
3324 }
3325}
3326
3327void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3328 if (!KmpRoutineEntryPtrTy) {
3329 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3330 ASTContext &C = CGM.getContext();
3331 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3332 FunctionProtoType::ExtProtoInfo EPI;
3333 KmpRoutineEntryPtrQTy = C.getPointerType(
3334 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3335 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3336 }
3337}
3338
3339QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
3340 // Make sure the type of the entry is already created. This is the type we
3341 // have to create:
3342 // struct __tgt_offload_entry{
3343 // void *addr; // Pointer to the offload entry info.
3344 // // (function or global)
3345 // char *name; // Name of the function or global.
3346 // size_t size; // Size of the entry info (0 if it a function).
3347 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
3348 // int32_t reserved; // Reserved, to use by the runtime library.
3349 // };
3350 if (TgtOffloadEntryQTy.isNull()) {
3351 ASTContext &C = CGM.getContext();
3352 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
3353 RD->startDefinition();
3354 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3355 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
3356 addFieldToRecordDecl(C, RD, C.getSizeType());
3357 addFieldToRecordDecl(
3358 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3359 addFieldToRecordDecl(
3360 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3361 RD->completeDefinition();
3362 RD->addAttr(PackedAttr::CreateImplicit(C));
3363 TgtOffloadEntryQTy = C.getRecordType(RD);
3364 }
3365 return TgtOffloadEntryQTy;
3366}
3367
3368namespace {
3369struct PrivateHelpersTy {
3370 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3371 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3372 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3373 PrivateElemInit(PrivateElemInit) {}
3374 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3375 const Expr *OriginalRef = nullptr;
3376 const VarDecl *Original = nullptr;
3377 const VarDecl *PrivateCopy = nullptr;
3378 const VarDecl *PrivateElemInit = nullptr;
3379 bool isLocalPrivate() const {
3380 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3381 }
3382};
3383typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3384} // anonymous namespace
3385
3386static bool isAllocatableDecl(const VarDecl *VD) {
3387 const VarDecl *CVD = VD->getCanonicalDecl();
3388 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3389 return false;
3390 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3391 // Use the default allocation.
3392 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
3393 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
3394 !AA->getAllocator());
3395}
3396
3397static RecordDecl *
3398createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3399 if (!Privates.empty()) {
3400 ASTContext &C = CGM.getContext();
3401 // Build struct .kmp_privates_t. {
3402 // /* private vars */
3403 // };
3404 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
3405 RD->startDefinition();
3406 for (const auto &Pair : Privates) {
3407 const VarDecl *VD = Pair.second.Original;
3408 QualType Type = VD->getType().getNonReferenceType();
3409 // If the private variable is a local variable with lvalue ref type,
3410 // allocate the pointer instead of the pointee type.
3411 if (Pair.second.isLocalPrivate()) {
3412 if (VD->getType()->isLValueReferenceType())
3413 Type = C.getPointerType(Type);
3414 if (isAllocatableDecl(VD))
3415 Type = C.getPointerType(Type);
3416 }
3417 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
3418 if (VD->hasAttrs()) {
3419 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3420 E(VD->getAttrs().end());
3421 I != E; ++I)
3422 FD->addAttr(*I);
3423 }
3424 }
3425 RD->completeDefinition();
3426 return RD;
3427 }
3428 return nullptr;
3429}
3430
3431static RecordDecl *
3432createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3433 QualType KmpInt32Ty,
3434 QualType KmpRoutineEntryPointerQTy) {
3435 ASTContext &C = CGM.getContext();
3436 // Build struct kmp_task_t {
3437 // void * shareds;
3438 // kmp_routine_entry_t routine;
3439 // kmp_int32 part_id;
3440 // kmp_cmplrdata_t data1;
3441 // kmp_cmplrdata_t data2;
3442 // For taskloops additional fields:
3443 // kmp_uint64 lb;
3444 // kmp_uint64 ub;
3445 // kmp_int64 st;
3446 // kmp_int32 liter;
3447 // void * reductions;
3448 // };
3449 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
3450 UD->startDefinition();
3451 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3452 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3453 UD->completeDefinition();
3454 QualType KmpCmplrdataTy = C.getRecordType(UD);
3455 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
3456 RD->startDefinition();
3457 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3458 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3459 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3460 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3461 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3462 if (isOpenMPTaskLoopDirective(Kind)) {
3463 QualType KmpUInt64Ty =
3464 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3465 QualType KmpInt64Ty =
3466 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3467 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3468 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3469 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3470 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3471 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3472 }
3473 RD->completeDefinition();
3474 return RD;
3475}
3476
3477static RecordDecl *
3478createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3479 ArrayRef<PrivateDataTy> Privates) {
3480 ASTContext &C = CGM.getContext();
3481 // Build struct kmp_task_t_with_privates {
3482 // kmp_task_t task_data;
3483 // .kmp_privates_t. privates;
3484 // };
3485 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3486 RD->startDefinition();
3487 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
3488 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3489 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
3490 RD->completeDefinition();
3491 return RD;
3492}
3493
3494/// Emit a proxy function which accepts kmp_task_t as the second
3495/// argument.
3496/// \code
3497/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3498/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3499/// For taskloops:
3500/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3501/// tt->reductions, tt->shareds);
3502/// return 0;
3503/// }
3504/// \endcode
3505static llvm::Function *
3506emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3507 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3508 QualType KmpTaskTWithPrivatesPtrQTy,
3509 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3510 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3511 llvm::Value *TaskPrivatesMap) {
3512 ASTContext &C = CGM.getContext();
3513 FunctionArgList Args;
3514 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3515 ImplicitParamDecl::Other);
3516 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3517 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3518 ImplicitParamDecl::Other);
3519 Args.push_back(&GtidArg);
3520 Args.push_back(&TaskTypeArg);
3521 const auto &TaskEntryFnInfo =
3522 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3523 llvm::FunctionType *TaskEntryTy =
3524 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3525 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
3526 auto *TaskEntry = llvm::Function::Create(
3527 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3528 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
3529 TaskEntry->setDoesNotRecurse();
3530 CodeGenFunction CGF(CGM);
3531 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3532 Loc, Loc);
3533
3534 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3535 // tt,
3536 // For taskloops:
3537 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3538 // tt->task_data.shareds);
3539 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3540 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3541 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3542 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3543 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3544 const auto *KmpTaskTWithPrivatesQTyRD =
3545 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3546 LValue Base =
3547 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3548 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
3549 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3550 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3551 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3552
3553 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3554 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3555 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3556 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3557 CGF.ConvertTypeForMem(SharedsPtrTy));
3558
3559 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3560 llvm::Value *PrivatesParam;
3561 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3562 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3563 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3564 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3565 } else {
3566 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3567 }
3568
3569 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3570 TaskPrivatesMap,
3571 CGF.Builder
3572 .CreatePointerBitCastOrAddrSpaceCast(
3573 TDBase.getAddress(CGF), CGF.VoidPtrTy)
3574 .getPointer()};
3575 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3576 std::end(CommonArgs));
3577 if (isOpenMPTaskLoopDirective(Kind)) {
3578 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3579 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3580 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3581 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3582 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3583 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3584 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3585 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3586 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3587 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3588 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3589 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3590 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3591 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3592 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3593 CallArgs.push_back(LBParam);
3594 CallArgs.push_back(UBParam);
3595 CallArgs.push_back(StParam);
3596 CallArgs.push_back(LIParam);
3597 CallArgs.push_back(RParam);
3598 }
3599 CallArgs.push_back(SharedsParam);
3600
3601 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3602 CallArgs);
3603 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3604 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3605 CGF.FinishFunction();
3606 return TaskEntry;
3607}
3608
3609static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3610 SourceLocation Loc,
3611 QualType KmpInt32Ty,
3612 QualType KmpTaskTWithPrivatesPtrQTy,
3613 QualType KmpTaskTWithPrivatesQTy) {
3614 ASTContext &C = CGM.getContext();
3615 FunctionArgList Args;
3616 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3617 ImplicitParamDecl::Other);
3618 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3619 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3620 ImplicitParamDecl::Other);
3621 Args.push_back(&GtidArg);
3622 Args.push_back(&TaskTypeArg);
3623 const auto &DestructorFnInfo =
3624 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3625 llvm::FunctionType *DestructorFnTy =
3626 CGM.getTypes().GetFunctionType(DestructorFnInfo);
3627 std::string Name =
3628 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
3629 auto *DestructorFn =
3630 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3631 Name, &CGM.getModule());
3632 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
3633 DestructorFnInfo);
3634 DestructorFn->setDoesNotRecurse();
3635 CodeGenFunction CGF(CGM);
3636 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3637 Args, Loc, Loc);
3638
3639 LValue Base = CGF.EmitLoadOfPointerLValue(
3640 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3641 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3642 const auto *KmpTaskTWithPrivatesQTyRD =
3643 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3644 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3645 Base = CGF.EmitLValueForField(Base, *FI);
3646 for (const auto *Field :
3647 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3648 if (QualType::DestructionKind DtorKind =
3649 Field->getType().isDestructedType()) {
3650 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3651 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
3652 }
3653 }
3654 CGF.FinishFunction();
3655 return DestructorFn;
3656}
3657
3658/// Emit a privates mapping function for correct handling of private and
3659/// firstprivate variables.
3660/// \code
3661/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3662/// **noalias priv1,..., <tyn> **noalias privn) {
3663/// *priv1 = &.privates.priv1;
3664/// ...;
3665/// *privn = &.privates.privn;
3666/// }
3667/// \endcode
3668static llvm::Value *
3669emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3670 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3671 ArrayRef<PrivateDataTy> Privates) {
3672 ASTContext &C = CGM.getContext();
3673 FunctionArgList Args;
3674 ImplicitParamDecl TaskPrivatesArg(
3675 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3676 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3677 ImplicitParamDecl::Other);
3678 Args.push_back(&TaskPrivatesArg);
3679 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3680 unsigned Counter = 1;
3681 for (const Expr *E : Data.PrivateVars) {
3682 Args.push_back(ImplicitParamDecl::Create(
3683 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3684 C.getPointerType(C.getPointerType(E->getType()))
3685 .withConst()
3686 .withRestrict(),
3687 ImplicitParamDecl::Other));
3688 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3689 PrivateVarsPos[VD] = Counter;
3690 ++Counter;
3691 }
3692 for (const Expr *E : Data.FirstprivateVars) {
3693 Args.push_back(ImplicitParamDecl::Create(
3694 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3695 C.getPointerType(C.getPointerType(E->getType()))
3696 .withConst()
3697 .withRestrict(),
3698 ImplicitParamDecl::Other));
3699 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3700 PrivateVarsPos[VD] = Counter;
3701 ++Counter;
3702 }
3703 for (const Expr *E : Data.LastprivateVars) {
3704 Args.push_back(ImplicitParamDecl::Create(
3705 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3706 C.getPointerType(C.getPointerType(E->getType()))
3707 .withConst()
3708 .withRestrict(),
3709 ImplicitParamDecl::Other));
3710 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3711 PrivateVarsPos[VD] = Counter;
3712 ++Counter;
3713 }
3714 for (const VarDecl *VD : Data.PrivateLocals) {
3715 QualType Ty = VD->getType().getNonReferenceType();
3716 if (VD->getType()->isLValueReferenceType())
3717 Ty = C.getPointerType(Ty);
3718 if (isAllocatableDecl(VD))
3719 Ty = C.getPointerType(Ty);
3720 Args.push_back(ImplicitParamDecl::Create(
3721 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3722 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3723 ImplicitParamDecl::Other));
3724 PrivateVarsPos[VD] = Counter;
3725 ++Counter;
3726 }
3727 const auto &TaskPrivatesMapFnInfo =
3728 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3729 llvm::FunctionType *TaskPrivatesMapTy =
3730 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
3731 std::string Name =
3732 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
3733 auto *TaskPrivatesMap = llvm::Function::Create(
3734 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3735 &CGM.getModule());
3736 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
3737 TaskPrivatesMapFnInfo);
3738 if (CGM.getLangOpts().Optimize) {
3739 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3740 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3741 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3742 }
3743 CodeGenFunction CGF(CGM);
3744 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3745 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3746
3747 // *privi = &.privates.privi;
3748 LValue Base = CGF.EmitLoadOfPointerLValue(
3749 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3750 TaskPrivatesArg.getType()->castAs<PointerType>());
3751 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
3752 Counter = 0;
3753 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3754 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3755 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3756 LValue RefLVal =
3757 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3758 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3759 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
3760 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3761 ++Counter;
3762 }
3763 CGF.FinishFunction();
3764 return TaskPrivatesMap;
3765}
3766
3767/// Emit initialization for private variables in task-based directives.
3768static void emitPrivatesInit(CodeGenFunction &CGF,
3769 const OMPExecutableDirective &D,
3770 Address KmpTaskSharedsPtr, LValue TDBase,
3771 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3772 QualType SharedsTy, QualType SharedsPtrTy,
3773 const OMPTaskDataTy &Data,
3774 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3775 ASTContext &C = CGF.getContext();
3776 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3777 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3778 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
3779 ? OMPD_taskloop
3780 : OMPD_task;
3781 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
3782 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3783 LValue SrcBase;
3784 bool IsTargetTask =
3785 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
3786 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
3787 // For target-based directives skip 3 firstprivate arrays BasePointersArray,
3788 // PointersArray and SizesArray. The original variables for these arrays are
3789 // not captured and we get their addresses explicitly.
3790 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3791 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3792 SrcBase = CGF.MakeAddrLValue(
3793 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3794 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
3795 SharedsTy);
3796 }
3797 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3798 for (const PrivateDataTy &Pair : Privates) {
3799 // Do not initialize private locals.
3800 if (Pair.second.isLocalPrivate()) {
3801 ++FI;
3802 continue;
3803 }
3804 const VarDecl *VD = Pair.second.PrivateCopy;
3805 const Expr *Init = VD->getAnyInitializer();
3806 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3807 !CGF.isTrivialInitializer(Init)))) {
3808 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3809 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3810 const VarDecl *OriginalVD = Pair.second.Original;
3811 // Check if the variable is the target-based BasePointersArray,
3812 // PointersArray or SizesArray.
3813 LValue SharedRefLValue;
3814 QualType Type = PrivateLValue.getType();
3815 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3816 if (IsTargetTask && !SharedField) {
3817 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3818 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3819 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3820 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3821 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3822 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3823 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
3824 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3824, __PRETTY_FUNCTION__))
;
3825 SharedRefLValue =
3826 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3827 } else if (ForDup) {
3828 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3829 SharedRefLValue = CGF.MakeAddrLValue(
3830 Address(SharedRefLValue.getPointer(CGF),
3831 C.getDeclAlign(OriginalVD)),
3832 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3833 SharedRefLValue.getTBAAInfo());
3834 } else if (CGF.LambdaCaptureFields.count(
3835 Pair.second.Original->getCanonicalDecl()) > 0 ||
3836 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) {
3837 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3838 } else {
3839 // Processing for implicitly captured variables.
3840 InlinedOpenMPRegionRAII Region(
3841 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3842 /*HasCancel=*/false);
3843 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3844 }
3845 if (Type->isArrayType()) {
3846 // Initialize firstprivate array.
3847 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3848 // Perform simple memcpy.
3849 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3850 } else {
3851 // Initialize firstprivate array using element-by-element
3852 // initialization.
3853 CGF.EmitOMPAggregateAssign(
3854 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
3855 Type,
3856 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3857 Address SrcElement) {
3858 // Clean up any temporaries needed by the initialization.
3859 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3860 InitScope.addPrivate(
3861 Elem, [SrcElement]() -> Address { return SrcElement; });
3862 (void)InitScope.Privatize();
3863 // Emit initialization for single element.
3864 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3865 CGF, &CapturesInfo);
3866 CGF.EmitAnyExprToMem(Init, DestElement,
3867 Init->getType().getQualifiers(),
3868 /*IsInitializer=*/false);
3869 });
3870 }
3871 } else {
3872 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3873 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
3874 return SharedRefLValue.getAddress(CGF);
3875 });
3876 (void)InitScope.Privatize();
3877 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3878 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3879 /*capturedByInit=*/false);
3880 }
3881 } else {
3882 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3883 }
3884 }
3885 ++FI;
3886 }
3887}
3888
3889/// Check if duplication function is required for taskloops.
3890static bool checkInitIsRequired(CodeGenFunction &CGF,
3891 ArrayRef<PrivateDataTy> Privates) {
3892 bool InitRequired = false;
3893 for (const PrivateDataTy &Pair : Privates) {
3894 if (Pair.second.isLocalPrivate())
3895 continue;
3896 const VarDecl *VD = Pair.second.PrivateCopy;
3897 const Expr *Init = VD->getAnyInitializer();
3898 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
3899 !CGF.isTrivialInitializer(Init));
3900 if (InitRequired)
3901 break;
3902 }
3903 return InitRequired;
3904}
3905
3906
3907/// Emit task_dup function (for initialization of
3908/// private/firstprivate/lastprivate vars and last_iter flag)
3909/// \code
3910/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3911/// lastpriv) {
3912/// // setup lastprivate flag
3913/// task_dst->last = lastpriv;
3914/// // could be constructor calls here...
3915/// }
3916/// \endcode
3917static llvm::Value *
3918emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3919 const OMPExecutableDirective &D,
3920 QualType KmpTaskTWithPrivatesPtrQTy,
3921 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3922 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3923 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3924 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3925 ASTContext &C = CGM.getContext();
3926 FunctionArgList Args;
3927 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3928 KmpTaskTWithPrivatesPtrQTy,
3929 ImplicitParamDecl::Other);
3930 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3931 KmpTaskTWithPrivatesPtrQTy,
3932 ImplicitParamDecl::Other);
3933 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
3934 ImplicitParamDecl::Other);
3935 Args.push_back(&DstArg);
3936 Args.push_back(&SrcArg);
3937 Args.push_back(&LastprivArg);
3938 const auto &TaskDupFnInfo =
3939 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3940 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
3941 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
3942 auto *TaskDup = llvm::Function::Create(
3943 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3944 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
3945 TaskDup->setDoesNotRecurse();
3946 CodeGenFunction CGF(CGM);
3947 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3948 Loc);
3949
3950 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3951 CGF.GetAddrOfLocalVar(&DstArg),
3952 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3953 // task_dst->liter = lastpriv;
3954 if (WithLastIter) {
3955 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3956 LValue Base = CGF.EmitLValueForField(
3957 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3958 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3959 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3960 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3961 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3962 }
3963
3964 // Emit initial values for private copies (if any).
3965 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3965, __PRETTY_FUNCTION__))
;
3966 Address KmpTaskSharedsPtr = Address::invalid();
3967 if (!Data.FirstprivateVars.empty()) {
3968 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3969 CGF.GetAddrOfLocalVar(&SrcArg),
3970 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3971 LValue Base = CGF.EmitLValueForField(
3972 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3973 KmpTaskSharedsPtr = Address(
3974 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3975 Base, *std::next(KmpTaskTQTyRD->field_begin(),
3976 KmpTaskTShareds)),
3977 Loc),
3978 CGM.getNaturalTypeAlignment(SharedsTy));
3979 }
3980 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3981 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3982 CGF.FinishFunction();
3983 return TaskDup;
3984}
3985
3986/// Checks if destructor function is required to be generated.
3987/// \return true if cleanups are required, false otherwise.
3988static bool
3989checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3990 ArrayRef<PrivateDataTy> Privates) {
3991 for (const PrivateDataTy &P : Privates) {
3992 if (P.second.isLocalPrivate())
3993 continue;
3994 QualType Ty = P.second.Original->getType().getNonReferenceType();
3995 if (Ty.isDestructedType())
3996 return true;
3997 }
3998 return false;
3999}
4000
4001namespace {
4002/// Loop generator for OpenMP iterator expression.
4003class OMPIteratorGeneratorScope final
4004 : public CodeGenFunction::OMPPrivateScope {
4005 CodeGenFunction &CGF;
4006 const OMPIteratorExpr *E = nullptr;
4007 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
4008 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
4009 OMPIteratorGeneratorScope() = delete;
4010 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
4011
4012public:
4013 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
4014 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
4015 if (!E)
4016 return;
4017 SmallVector<llvm::Value *, 4> Uppers;
4018 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4019 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
4020 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
4021 addPrivate(VD, [&CGF, VD]() {
4022 return CGF.CreateMemTemp(VD->getType(), VD->getName());
4023 });
4024 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4025 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() {
4026 return CGF.CreateMemTemp(HelperData.CounterVD->getType(),
4027 "counter.addr");
4028 });
4029 }
4030 Privatize();
4031
4032 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4033 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4034 LValue CLVal =
4035 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
4036 HelperData.CounterVD->getType());
4037 // Counter = 0;
4038 CGF.EmitStoreOfScalar(
4039 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0),
4040 CLVal);
4041 CodeGenFunction::JumpDest &ContDest =
4042 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
4043 CodeGenFunction::JumpDest &ExitDest =
4044 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
4045 // N = <number-of_iterations>;
4046 llvm::Value *N = Uppers[I];
4047 // cont:
4048 // if (Counter < N) goto body; else goto exit;
4049 CGF.EmitBlock(ContDest.getBlock());
4050 auto *CVal =
4051 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
4052 llvm::Value *Cmp =
4053 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
4054 ? CGF.Builder.CreateICmpSLT(CVal, N)
4055 : CGF.Builder.CreateICmpULT(CVal, N);
4056 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
4057 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
4058 // body:
4059 CGF.EmitBlock(BodyBB);
4060 // Iteri = Begini + Counter * Stepi;
4061 CGF.EmitIgnoredExpr(HelperData.Update);
4062 }
4063 }
4064 ~OMPIteratorGeneratorScope() {
4065 if (!E)
4066 return;
4067 for (unsigned I = E->numOfIterators(); I > 0; --I) {
4068 // Counter = Counter + 1;
4069 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);
4070 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
4071 // goto cont;
4072 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
4073 // exit:
4074 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
4075 }
4076 }
4077};
4078} // namespace
4079
4080static std::pair<llvm::Value *, llvm::Value *>
4081getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
4082 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
4083 llvm::Value *Addr;
4084 if (OASE) {
4085 const Expr *Base = OASE->getBase();
4086 Addr = CGF.EmitScalarExpr(Base);
4087 } else {
4088 Addr = CGF.EmitLValue(E).getPointer(CGF);
4089 }
4090 llvm::Value *SizeVal;
4091 QualType Ty = E->getType();
4092 if (OASE) {
4093 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
4094 for (const Expr *SE : OASE->getDimensions()) {
4095 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
4096 Sz = CGF.EmitScalarConversion(
4097 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
4098 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
4099 }
4100 } else if (const auto *ASE =
4101 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
4102 LValue UpAddrLVal =
4103 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
4104 llvm::Value *UpAddr =
4105 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
4106 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
4107 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
4108 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4109 } else {
4110 SizeVal = CGF.getTypeSize(Ty);
4111 }
4112 return std::make_pair(Addr, SizeVal);
4113}
4114
4115/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4116static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
4117 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);
4118 if (KmpTaskAffinityInfoTy.isNull()) {
4119 RecordDecl *KmpAffinityInfoRD =
4120 C.buildImplicitRecord("kmp_task_affinity_info_t");
4121 KmpAffinityInfoRD->startDefinition();
4122 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());
4123 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
4124 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
4125 KmpAffinityInfoRD->completeDefinition();
4126 KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
4127 }
4128}
4129
4130CGOpenMPRuntime::TaskResultTy
4131CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4132 const OMPExecutableDirective &D,
4133 llvm::Function *TaskFunction, QualType SharedsTy,
4134 Address Shareds, const OMPTaskDataTy &Data) {
4135 ASTContext &C = CGM.getContext();
4136 llvm::SmallVector<PrivateDataTy, 4> Privates;
4137 // Aggregate privates and sort them by the alignment.
4138 const auto *I = Data.PrivateCopies.begin();
4139 for (const Expr *E : Data.PrivateVars) {
4140 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4141 Privates.emplace_back(
4142 C.getDeclAlign(VD),
4143 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4144 /*PrivateElemInit=*/nullptr));
4145 ++I;
4146 }
4147 I = Data.FirstprivateCopies.begin();
4148 const auto *IElemInitRef = Data.FirstprivateInits.begin();
4149 for (const Expr *E : Data.FirstprivateVars) {
4150 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4151 Privates.emplace_back(
4152 C.getDeclAlign(VD),
4153 PrivateHelpersTy(
4154 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4155 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4156 ++I;
4157 ++IElemInitRef;
4158 }
4159 I = Data.LastprivateCopies.begin();
4160 for (const Expr *E : Data.LastprivateVars) {
4161 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4162 Privates.emplace_back(
4163 C.getDeclAlign(VD),
4164 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4165 /*PrivateElemInit=*/nullptr));
4166 ++I;
4167 }
4168 for (const VarDecl *VD : Data.PrivateLocals) {
4169 if (isAllocatableDecl(VD))
4170 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));
4171 else
4172 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));
4173 }
4174 llvm::stable_sort(Privates,
4175 [](const PrivateDataTy &L, const PrivateDataTy &R) {
4176 return L.first > R.first;
4177 });
4178 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4179 // Build type kmp_routine_entry_t (if not built yet).
4180 emitKmpRoutineEntryT(KmpInt32Ty);
4181 // Build type kmp_task_t (if not built yet).
4182 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4183 if (SavedKmpTaskloopTQTy.isNull()) {
4184 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4185 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4186 }
4187 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4188 } else {
4189 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4192, __PRETTY_FUNCTION__))
4190 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4192, __PRETTY_FUNCTION__))
4191 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4192, __PRETTY_FUNCTION__))
4192 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4192, __PRETTY_FUNCTION__))
;
4193 if (SavedKmpTaskTQTy.isNull()) {
4194 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4195 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4196 }
4197 KmpTaskTQTy = SavedKmpTaskTQTy;
4198 }
4199 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4200 // Build particular struct kmp_task_t for the given task.
4201 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4202 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4203 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4204 QualType KmpTaskTWithPrivatesPtrQTy =
4205 C.getPointerType(KmpTaskTWithPrivatesQTy);
4206 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4207 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4208 KmpTaskTWithPrivatesTy->getPointerTo();
4209 llvm::Value *KmpTaskTWithPrivatesTySize =
4210 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4211 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4212
4213 // Emit initial values for private copies (if any).
4214 llvm::Value *TaskPrivatesMap = nullptr;
4215 llvm::Type *TaskPrivatesMapTy =
4216 std::next(TaskFunction->arg_begin(), 3)->getType();
4217 if (!Privates.empty()) {
4218 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4219 TaskPrivatesMap =
4220 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);
4221 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4222 TaskPrivatesMap, TaskPrivatesMapTy);
4223 } else {
4224 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4225 cast<llvm::PointerType>(TaskPrivatesMapTy));
4226 }
4227 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4228 // kmp_task_t *tt);
4229 llvm::Function *TaskEntry = emitProxyTaskFunction(
4230 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4231 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4232 TaskPrivatesMap);
4233
4234 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4235 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4236 // kmp_routine_entry_t *task_entry);
4237 // Task flags. Format is taken from
4238 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
4239 // description of kmp_tasking_flags struct.
4240 enum {
4241 TiedFlag = 0x1,
4242 FinalFlag = 0x2,
4243 DestructorsFlag = 0x8,
4244 PriorityFlag = 0x20,
4245 DetachableFlag = 0x40,
4246 };
4247 unsigned Flags = Data.Tied ? TiedFlag : 0;
4248 bool NeedsCleanup = false;
4249 if (!Privates.empty()) {
4250 NeedsCleanup =
4251 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4252 if (NeedsCleanup)
4253 Flags = Flags | DestructorsFlag;
4254 }
4255 if (Data.Priority.getInt())
4256 Flags = Flags | PriorityFlag;
4257 if (D.hasClausesOfKind<OMPDetachClause>())
4258 Flags = Flags | DetachableFlag;
4259 llvm::Value *TaskFlags =
4260 Data.Final.getPointer()
4261 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4262 CGF.Builder.getInt32(FinalFlag),
4263 CGF.Builder.getInt32(/*C=*/0))
4264 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4265 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4266 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4267 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4268 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4269 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4270 TaskEntry, KmpRoutineEntryPtrTy)};
4271 llvm::Value *NewTask;
4272 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4273 // Check if we have any device clause associated with the directive.
4274 const Expr *Device = nullptr;
4275 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4276 Device = C->getDevice();
4277 // Emit device ID if any otherwise use default value.
4278 llvm::Value *DeviceID;
4279 if (Device)
4280 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
4281 CGF.Int64Ty, /*isSigned=*/true);
4282 else
4283 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
4284 AllocArgs.push_back(DeviceID);
4285 NewTask = CGF.EmitRuntimeCall(
4286 OMPBuilder.getOrCreateRuntimeFunction(
4287 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4288 AllocArgs);
4289 } else {
4290 NewTask =
4291 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4292 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4293 AllocArgs);
4294 }
4295 // Emit detach clause initialization.
4296 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4297 // task_descriptor);
4298 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4299 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4300 LValue EvtLVal = CGF.EmitLValue(Evt);
4301
4302 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4303 // int gtid, kmp_task_t *task);
4304 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
4305 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
4306 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
4307 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4308 OMPBuilder.getOrCreateRuntimeFunction(
4309 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4310 {Loc, Tid, NewTask});
4311 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
4312 Evt->getExprLoc());
4313 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
4314 }
4315 // Process affinity clauses.
4316 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4317 // Process list of affinity data.
4318 ASTContext &C = CGM.getContext();
4319 Address AffinitiesArray = Address::invalid();
4320 // Calculate number of elements to form the array of affinity data.
4321 llvm::Value *NumOfElements = nullptr;
4322 unsigned NumAffinities = 0;
4323 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4324 if (const Expr *Modifier = C->getModifier()) {
4325 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4326 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4327 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4328 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4329 NumOfElements =
4330 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4331 }
4332 } else {
4333 NumAffinities += C->varlist_size();
4334 }
4335 }
4336 getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy);
4337 // Fields ids in kmp_task_affinity_info record.
4338 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4339
4340 QualType KmpTaskAffinityInfoArrayTy;
4341 if (NumOfElements) {
4342 NumOfElements = CGF.Builder.CreateNUWAdd(
4343 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
4344 OpaqueValueExpr OVE(
4345 Loc,
4346 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),
4347 VK_RValue);
4348 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4349 RValue::get(NumOfElements));
4350 KmpTaskAffinityInfoArrayTy =
4351 C.getVariableArrayType(KmpTaskAffinityInfoTy, &OVE, ArrayType::Normal,
4352 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4353 // Properly emit variable-sized array.
4354 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,
4355 ImplicitParamDecl::Other);
4356 CGF.EmitVarDecl(*PD);
4357 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
4358 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4359 /*isSigned=*/false);
4360 } else {
4361 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4362 KmpTaskAffinityInfoTy,
4363 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,
4364 ArrayType::Normal, /*IndexTypeQuals=*/0);
4365 AffinitiesArray =
4366 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
4367 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
4368 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,
4369 /*isSigned=*/false);
4370 }
4371
4372 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4373 // Fill array by elements without iterators.
4374 unsigned Pos = 0;
4375 bool HasIterator = false;
4376 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4377 if (C->getModifier()) {
4378 HasIterator = true;
4379 continue;
4380 }
4381 for (const Expr *E : C->varlists()) {
4382 llvm::Value *Addr;
4383 llvm::Value *Size;
4384 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4385 LValue Base =
4386 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
4387 KmpTaskAffinityInfoTy);
4388 // affs[i].base_addr = &<Affinities[i].second>;
4389 LValue BaseAddrLVal = CGF.EmitLValueForField(
4390 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4391 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4392 BaseAddrLVal);
4393 // affs[i].len = sizeof(<Affinities[i].second>);
4394 LValue LenLVal = CGF.EmitLValueForField(
4395 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4396 CGF.EmitStoreOfScalar(Size, LenLVal);
4397 ++Pos;
4398 }
4399 }
4400 LValue PosLVal;
4401 if (HasIterator) {
4402 PosLVal = CGF.MakeAddrLValue(
4403 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
4404 C.getSizeType());
4405 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4406 }
4407 // Process elements with iterators.
4408 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4409 const Expr *Modifier = C->getModifier();
4410 if (!Modifier)
4411 continue;
4412 OMPIteratorGeneratorScope IteratorScope(
4413 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
4414 for (const Expr *E : C->varlists()) {
4415 llvm::Value *Addr;
4416 llvm::Value *Size;
4417 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4418 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4419 LValue Base = CGF.MakeAddrLValue(
4420 Address(CGF.Builder.CreateGEP(AffinitiesArray.getPointer(), Idx),
4421 AffinitiesArray.getAlignment()),
4422 KmpTaskAffinityInfoTy);
4423 // affs[i].base_addr = &<Affinities[i].second>;
4424 LValue BaseAddrLVal = CGF.EmitLValueForField(
4425 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4426 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4427 BaseAddrLVal);
4428 // affs[i].len = sizeof(<Affinities[i].second>);
4429 LValue LenLVal = CGF.EmitLValueForField(
4430 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4431 CGF.EmitStoreOfScalar(Size, LenLVal);
4432 Idx = CGF.Builder.CreateNUWAdd(
4433 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4434 CGF.EmitStoreOfScalar(Idx, PosLVal);
4435 }
4436 }
4437 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4438 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4439 // naffins, kmp_task_affinity_info_t *affin_list);
4440 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4441 llvm::Value *GTid = getThreadID(CGF, Loc);
4442 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4443 AffinitiesArray.getPointer(), CGM.VoidPtrTy);
4444 // FIXME: Emit the function and ignore its result for now unless the
4445 // runtime function is properly implemented.
4446 (void)CGF.EmitRuntimeCall(
4447 OMPBuilder.getOrCreateRuntimeFunction(
4448 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4449 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4450 }
4451 llvm::Value *NewTaskNewTaskTTy =
4452 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4453 NewTask, KmpTaskTWithPrivatesPtrTy);
4454 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4455 KmpTaskTWithPrivatesQTy);
4456 LValue TDBase =
4457 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4458 // Fill the data in the resulting kmp_task_t record.
4459 // Copy shareds if there are any.
4460 Address KmpTaskSharedsPtr = Address::invalid();
4461 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4462 KmpTaskSharedsPtr =
4463 Address(CGF.EmitLoadOfScalar(
4464 CGF.EmitLValueForField(
4465 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4466 KmpTaskTShareds)),
4467 Loc),
4468 CGM.getNaturalTypeAlignment(SharedsTy));
4469 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4470 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4471 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4472 }
4473 // Emit initial values for private copies (if any).
4474 TaskResultTy Result;
4475 if (!Privates.empty()) {
4476 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4477 SharedsTy, SharedsPtrTy, Data, Privates,
4478 /*ForDup=*/false);
4479 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4480 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4481 Result.TaskDupFn = emitTaskDupFunction(
4482 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4483 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4484 /*WithLastIter=*/!Data.LastprivateVars.empty());
4485 }
4486 }
4487 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4488 enum { Priority = 0, Destructors = 1 };
4489 // Provide pointer to function with destructors for privates.
4490 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4491 const RecordDecl *KmpCmplrdataUD =
4492 (*FI)->getType()->getAsUnionType()->getDecl();
4493 if (NeedsCleanup) {
4494 llvm::Value *DestructorFn = emitDestructorsFunction(
4495 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4496 KmpTaskTWithPrivatesQTy);
4497 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4498 LValue DestructorsLV = CGF.EmitLValueForField(
4499 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4500 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4501 DestructorFn, KmpRoutineEntryPtrTy),
4502 DestructorsLV);
4503 }
4504 // Set priority.
4505 if (Data.Priority.getInt()) {
4506 LValue Data2LV = CGF.EmitLValueForField(
4507 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4508 LValue PriorityLV = CGF.EmitLValueForField(
4509 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4510 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4511 }
4512 Result.NewTask = NewTask;
4513 Result.TaskEntry = TaskEntry;
4514 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4515 Result.TDBase = TDBase;
4516 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4517 return Result;
4518}
4519
4520namespace {
4521/// Dependence kind for RTL.
4522enum RTLDependenceKindTy {
4523 DepIn = 0x01,
4524 DepInOut = 0x3,
4525 DepMutexInOutSet = 0x4
4526};
4527/// Fields ids in kmp_depend_info record.
4528enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4529} // namespace
4530
4531/// Translates internal dependency kind into the runtime kind.
4532static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4533 RTLDependenceKindTy DepKind;
4534 switch (K) {
4535 case OMPC_DEPEND_in:
4536 DepKind = DepIn;
4537 break;
4538 // Out and InOut dependencies must use the same code.
4539 case OMPC_DEPEND_out:
4540 case OMPC_DEPEND_inout:
4541 DepKind = DepInOut;
4542 break;
4543 case OMPC_DEPEND_mutexinoutset:
4544 DepKind = DepMutexInOutSet;
4545 break;
4546 case OMPC_DEPEND_source:
4547 case OMPC_DEPEND_sink:
4548 case OMPC_DEPEND_depobj:
4549 case OMPC_DEPEND_unknown:
4550 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4550)
;
4551 }
4552 return DepKind;
4553}
4554
4555/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4556static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4557 QualType &FlagsTy) {
4558 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4559 if (KmpDependInfoTy.isNull()) {
4560 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4561 KmpDependInfoRD->startDefinition();
4562 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4563 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4564 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4565 KmpDependInfoRD->completeDefinition();
4566 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
4567 }
4568}
4569
4570std::pair<llvm::Value *, LValue>
4571CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4572 SourceLocation Loc) {
4573 ASTContext &C = CGM.getContext();
4574 QualType FlagsTy;
4575 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4576 RecordDecl *KmpDependInfoRD =
4577 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4578 LValue Base = CGF.EmitLoadOfPointerLValue(
4579 DepobjLVal.getAddress(CGF),
4580 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4581 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4582 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4583 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4584 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4585 Base.getTBAAInfo());
4586 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4587 Addr.getPointer(),
4588 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4589 LValue NumDepsBase = CGF.MakeAddrLValue(
4590 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4591 Base.getBaseInfo(), Base.getTBAAInfo());
4592 // NumDeps = deps[i].base_addr;
4593 LValue BaseAddrLVal = CGF.EmitLValueForField(
4594 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4595 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4596 return std::make_pair(NumDeps, Base);
4597}
4598
4599static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4600 llvm::PointerUnion<unsigned *, LValue *> Pos,
4601 const OMPTaskDataTy::DependData &Data,
4602 Address DependenciesArray) {
4603 CodeGenModule &CGM = CGF.CGM;
4604 ASTContext &C = CGM.getContext();
4605 QualType FlagsTy;
4606 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4607 RecordDecl *KmpDependInfoRD =
4608 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4609 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4610
4611 OMPIteratorGeneratorScope IteratorScope(
4612 CGF, cast_or_null<OMPIteratorExpr>(
4613 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4614 : nullptr));
4615 for (const Expr *E : Data.DepExprs) {
4616 llvm::Value *Addr;
4617 llvm::Value *Size;
4618 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4619 LValue Base;
4620 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4621 Base = CGF.MakeAddrLValue(
4622 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4623 } else {
4624 LValue &PosLVal = *Pos.get<LValue *>();
4625 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4626 Base = CGF.MakeAddrLValue(
4627 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Idx),
4628 DependenciesArray.getAlignment()),
4629 KmpDependInfoTy);
4630 }
4631 // deps[i].base_addr = &<Dependencies[i].second>;
4632 LValue BaseAddrLVal = CGF.EmitLValueForField(
4633 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4634 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4635 BaseAddrLVal);
4636 // deps[i].len = sizeof(<Dependencies[i].second>);
4637 LValue LenLVal = CGF.EmitLValueForField(
4638 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
4639 CGF.EmitStoreOfScalar(Size, LenLVal);
4640 // deps[i].flags = <Dependencies[i].first>;
4641 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);
4642 LValue FlagsLVal = CGF.EmitLValueForField(
4643 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
4644 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
4645 FlagsLVal);
4646 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4647 ++(*P);
4648 } else {
4649 LValue &PosLVal = *Pos.get<LValue *>();
4650 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4651 Idx = CGF.Builder.CreateNUWAdd(Idx,
4652 llvm::ConstantInt::get(Idx->getType(), 1));
4653 CGF.EmitStoreOfScalar(Idx, PosLVal);
4654 }
4655 }
4656}
4657
4658static SmallVector<llvm::Value *, 4>
4659emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4660 const OMPTaskDataTy::DependData &Data) {
4661 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
4662 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4662, __PRETTY_FUNCTION__))
;
4663 SmallVector<llvm::Value *, 4> Sizes;
4664 SmallVector<LValue, 4> SizeLVals;
4665 ASTContext &C = CGF.getContext();
4666 QualType FlagsTy;
4667 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4668 RecordDecl *KmpDependInfoRD =
4669 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4670 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4671 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4672 {
4673 OMPIteratorGeneratorScope IteratorScope(
4674 CGF, cast_or_null<OMPIteratorExpr>(
4675 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4676 : nullptr));
4677 for (const Expr *E : Data.DepExprs) {
4678 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4679 LValue Base = CGF.EmitLoadOfPointerLValue(
4680 DepobjLVal.getAddress(CGF),
4681 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4682 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4683 Base.getAddress(CGF), KmpDependInfoPtrT);
4684 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4685 Base.getTBAAInfo());
4686 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4687 Addr.getPointer(),
4688 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4689 LValue NumDepsBase = CGF.MakeAddrLValue(
4690 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4691 Base.getBaseInfo(), Base.getTBAAInfo());
4692 // NumDeps = deps[i].base_addr;
4693 LValue BaseAddrLVal = CGF.EmitLValueForField(
4694 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4695 llvm::Value *NumDeps =
4696 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4697 LValue NumLVal = CGF.MakeAddrLValue(
4698 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4699 C.getUIntPtrType());
4700 CGF.InitTempAlloca(NumLVal.getAddress(CGF),
4701 llvm::ConstantInt::get(CGF.IntPtrTy, 0));
4702 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4703 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4704 CGF.EmitStoreOfScalar(Add, NumLVal);
4705 SizeLVals.push_back(NumLVal);
4706 }
4707 }
4708 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4709 llvm::Value *Size =
4710 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4711 Sizes.push_back(Size);
4712 }
4713 return Sizes;
4714}
4715
4716static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4717 LValue PosLVal,
4718 const OMPTaskDataTy::DependData &Data,
4719 Address DependenciesArray) {
4720 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4721, __PRETTY_FUNCTION__))
4721 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20200915100651+00ba1a3de7f/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4721, __PRETTY_FUNCTION__))
;
4722 ASTContext &C = CGF.getContext();
4723 QualType FlagsTy;
4724 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4725 RecordDecl *KmpDependInfoRD =
4726 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4727 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4728 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4729 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4730 {
4731 OMPIteratorGeneratorScope IteratorScope(
4732 CGF, cast_or_null<OMPIteratorExpr>(
4733 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4734 : nullptr));
4735 for (unsigned I = 0, End = Data.DepExprs.size(); I < End; ++I) {
4736 const Expr *E = Data.DepExprs[I];
4737 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4738 LValue Base = CGF.EmitLoadOfPointerLValue(
4739 DepobjLVal.getAddress(CGF),
4740 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4741 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4742 Base.getAddress(CGF), KmpDependInfoPtrT);
4743 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4744 Base.getTBAAInfo());
4745
4746 // Get number of elements in a single depobj.
4747 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4748 Addr.getPointer(),
4749 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4750 LValue NumDepsBase = CGF.MakeAddrLValue(
4751 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4752 Base.getBaseInfo(), Base.getTBAAInfo());
4753 // NumDeps = deps[i].base_addr;
4754 LValue BaseAddrLVal = CGF.EmitLValueForField(
4755 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4756 llvm::Value *NumDeps =
4757 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4758
4759 // memcopy dependency data.
4760 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4761 ElSize,
4762 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4763 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4764 Address DepAddr =
4765 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Pos),
4766 DependenciesArray.getAlignment());
4767 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
4768
4769 // Increase pos.
4770 // pos += size;
4771 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4772 CGF.EmitStoreOfScalar(Add, PosLVal);
4773 }
4774 }
4775}
4776
4777std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4778 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4779 SourceLocation Loc) {
4780 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {
4781 return D.DepExprs.empty();
4782 }))
4783 return std::make_pair(nullptr, Address::invalid());
4784 // Process list of dependencies.
4785 ASTContext &C = CGM.getContext();
4786 Address DependenciesArray = Address::invalid();
4787 llvm::Value *NumOfElements = nullptr;
4788 unsigned NumDependencies = std::accumulate(
4789 Dependencies.begin(), Dependencies.end(), 0,
4790 [](unsigned V, const OMPTaskDataTy::DependData &D) {
4791 return D.DepKind == OMPC_DEPEND_depobj
4792 ? V
4793 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4794 });
4795 QualType FlagsTy;
4796 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4797 bool HasDepobjDeps = false;
4798 bool HasRegularWithIterators = false;
4799 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4800 llvm::Value *NumOfRegularWithIterators =
4801 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4802 // Calculate number of depobj dependecies and regular deps with the iterators.
4803 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4804 if (D.DepKind == OMPC_DEPEND_depobj) {
4805 SmallVector<llvm::Value *, 4> Sizes =
4806 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4807 for (llvm::Value *Size : Sizes) {
4808 NumOfDepobjElements =
4809 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4810 }
4811 HasDepobjDeps = true;
4812 continue;
4813 }
4814 // Include number of iterations, if any.
4815 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {
4816 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4817 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4818 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4819 NumOfRegularWithIterators =
4820 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz);
4821 }
4822 HasRegularWithIterators = true;
4823 continue;
4824 }
4825 }
4826
4827 QualType KmpDependInfoArrayTy;
4828 if (HasDepobjDeps || HasRegularWithIterators) {
4829 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,
4830 /*isSigned=*/false);
4831 if (HasDepobjDeps) {
4832 NumOfElements =
4833 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4834 }
4835 if (HasRegularWithIterators) {
4836 NumOfElements =
4837 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4838 }
4839 OpaqueValueExpr OVE(Loc,
4840 C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4841 VK_RValue);
4842 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4843 RValue::get(NumOfElements));
4844 KmpDependInfoArrayTy =
4845 C.getVariableArrayType(KmpDependInfoTy, &OVE, ArrayType::Normal,
4846 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4847 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4848 // Properly emit variable-sized array.
4849 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,
4850 ImplicitParamDecl::Other);
4851 CGF.EmitVarDecl(*PD);
4852 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4853 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4854 /*isSigned=*/false);
4855 } else {
4856 KmpDependInfoArrayTy = C.getConstantArrayType(
4857 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,
4858 ArrayType::Normal, /*IndexTypeQuals=*/0);
4859 DependenciesArray =
4860 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4861 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4862 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,
4863 /*isSigned=*/false);
4864 }
4865 unsigned Pos = 0;
4866 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4867 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4868 Dependencies[I].IteratorExpr)
4869 continue;
4870 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4871 DependenciesArray);
4872 }
4873 // Copy regular dependecies with iterators.
4874 LValue PosLVal = CGF.MakeAddrLValue(
4875 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4876 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4877 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4878 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4879 !Dependencies[I].IteratorExpr)
4880 continue;
4881 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4882 DependenciesArray);
4883 }
4884 // Copy final depobj arrays without iterators.
4885 if (HasDepobjDeps) {
4886 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4887 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4888 continue;
4889 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4890 DependenciesArray);
4891 }
4892 }
4893 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4894 DependenciesArray, CGF.VoidPtrTy);
4895 return std::make_pair(NumOfElements, DependenciesArray);
4896}
4897
4898Address CGOpenMPRuntime::emitDepobjDependClause(
4899 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4900 SourceLocation Loc) {
4901 if (Dependencies.DepExprs.empty())
4902 return Address::invalid();
4903 // Process list of dependencies.
4904 ASTContext &C = CGM.getContext();
4905 Address DependenciesArray = Address::invalid();
4906 unsigned NumDependencies = Dependencies.DepExprs.size();
4907 QualType FlagsTy;
4908 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4909 RecordDecl *KmpDependInfoRD =
4910 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4911
4912 llvm::Value *Size;
4913 // Define type kmp_depend_info[<Dependencies.size()>];
4914 // For depobj reserve one extra element to store the number of elements.
4915 // It is required to handle depobj(x) update(in) construct.
4916 // kmp_depend_info[<Dependencies.size()>] deps;
4917 llvm::Value *NumDepsVal;
4918 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4919 if (const auto *IE =
4920 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {
4921 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4922 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4923 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4924 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4925 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4926 }
4927 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4928 NumDepsVal);
4929 CharUnits SizeInBytes =
4930 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4931 llvm::Value *RecSize = CGM.getSize(SizeInBytes);
4932 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4933 NumDepsVal =
4934 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4935 } else {
4936 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4937 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4938 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
4939 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4940 Size = CGM.getSize(Sz.alignTo(Align));
4941 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4942 }
4943 // Need to allocate on the dynamic memory.
4944 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4945 // Use default allocator.
4946 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4947 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4948
4949 llvm::Value *Addr =
4950 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4951 CGM.getModule(), OMPRTL___kmpc_alloc),
4952 Args, ".dep.arr.addr");
4953 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4954 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo());
4955 DependenciesArray = Address(Addr, Align);
4956 // Write number of elements in the first element of array for depobj.
4957 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4958 // deps[i].base_addr = NumDependencies;
4959 LValue BaseAddrLVal = CGF.EmitLValueForField(
4960 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4961 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4962 llvm::PointerUnion<unsigned *, LValue *> Pos;
4963 unsigned Idx = 1;
4964 LValue PosLVal;
4965 if (Dependencies.IteratorExpr) {
4966 PosLVal = CGF.MakeAddrLValue(
4967 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4968 C.getSizeType());
4969 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4970 /*IsInit=*/true);
4971 Pos = &PosLVal;
4972 } else {
4973 Pos = &Idx;
4974 }
4975 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4976 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4977 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy);
4978 return DependenciesArray;
4979}
4980
4981void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4982 SourceLocation Loc) {
4983 ASTContext &C = CGM.getContext();
4984 QualType FlagsTy;
4985 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4986 LValue Base = CGF.EmitLoadOfPointerLValue(
4987 DepobjLVal.getAddress(CGF),
4988 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4989 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4990 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4991 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4992 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4993 Addr.getPointer(),
4994 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4995 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
4996 CGF.VoidPtrTy);
4997 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4998 // Use default allocator.
4999 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5000 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
5001
5002 // _kmpc_free(gtid, addr, nullptr);
5003 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5004 CGM.getModule(), OMPRTL___kmpc_free),
5005 Args);
5006}
5007
5008void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
5009 OpenMPDependClauseKind NewDepKind,
5010 SourceLocation Loc) {
5011 ASTContext &C = CGM.getContext();
5012 QualType FlagsTy;
5013 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5014 RecordDecl *KmpDependInfoRD =
5015 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5016 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5017 llvm::Value *NumDeps;
5018 LValue Base;
5019 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
5020
5021 Address Begin = Base.getAddress(CGF);
5022 // Cast from pointer to array type to pointer to single element.
5023 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps);
5024 // The basic structure here is a while-do loop.
5025 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
5026 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
5027 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5028 CGF.EmitBlock(BodyBB);
5029 llvm::PHINode *ElementPHI =
5030 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
5031 ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
5032 Begin = Address(ElementPHI, Begin.getAlignment());
5033 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
5034 Base.getTBAAInfo());
5035 // deps[i].flags = NewDepKind;
5036 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
5037 LValue FlagsLVal = CGF.EmitLValueForField(
5038 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5039 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5040 FlagsLVal);
5041
5042 // Shift the address forward by one element.
5043 Address ElementNext =
5044 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
5045 ElementPHI->addIncoming(ElementNext.getPointer(),
5046 CGF.Builder.GetInsertBlock());
5047 llvm::Value *IsEmpty =
5048 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
5049 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5050 // Done.
5051 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5052}
5053
5054void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5055 const OMPExecutableDirective &D,
5056 llvm::Function *TaskFunction,
5057 QualType SharedsTy, Address Shareds,
5058 const Expr *IfCond,
5059 const OMPTaskDataTy &Data) {
5060 if (!CGF.HaveInsertPoint())
5061 return;
5062
5063 TaskResultTy Result =
5064 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5065 llvm::Value *NewTask = Result.NewTask;
5066 llvm::Function *TaskEntry = Result.TaskEntry;
5067 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5068 LValue TDBase = Result.TDBase;
5069 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5070 // Process list of dependences.
5071 Address DependenciesArray = Address::invalid();
5072 llvm::Value *NumOfElements;
5073 std::tie(NumOfElements, DependenciesArray) =
5074 emitDependClause(CGF, Data.Dependences, Loc);
5075
5076 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5077 // libcall.
5078 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5079 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5080 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5081 // list is not empty
5082 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5083 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5084 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5085 llvm::Value *DepTaskArgs[7];
5086 if (!Data.Dependences.empty()) {
5087 DepTaskArgs[0] = UpLoc;
5088 DepTaskArgs[1] = ThreadID;
5089 DepTaskArgs[2] = NewTask;
5090 DepTaskArgs[3] = NumOfElements;
5091 DepTaskArgs[4] = DependenciesArray.getPointer();
5092 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5093 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5094 }
5095 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
5096 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5097 if (!Data.Tied) {
5098 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5099 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5100 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5101 }
5102 if (!Data.Dependences.empty()) {
5103 CGF.EmitRuntimeCall(
5104 OMPBuilder.getOrCreateRuntimeFunction(
5105 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
5106 DepTaskArgs);
5107 } else {
5108 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5109 CGM.getModule(), OMPRTL___kmpc_omp_task),
5110 TaskArgs);
5111 }
5112 // Check if parent region is untied and build return for untied task;
5113 if (auto *Region =
5114 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5115 Region->emitUntiedSwitch(CGF);
5116 };
5117
5118 llvm::Value *DepWaitTaskArgs[6];
5119 if (!Data.Dependences.empty()) {
5120 DepWaitTaskArgs[0] = UpLoc;
5121 DepWaitTaskArgs[1] = ThreadID;
5122 DepWaitTaskArgs[2] = NumOfElements;
5123 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5124 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5125 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5126 }
5127 auto &M = CGM.getModule();
5128 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
5129 TaskEntry, &Data, &DepWaitTaskArgs,
5130 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5131 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5132 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5133 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5134 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5135 // is specified.
5136 if (!Data.Dependences.empty())
5137 CGF.EmitRuntimeCall(
5138 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_wait_deps),
5139 DepWaitTaskArgs);
5140 // Call proxy_task_entry(gtid, new_task);
5141 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5142 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5143 Action.Enter(CGF);
5144 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5145 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5146 OutlinedFnArgs);
5147 };
5148
5149 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5150 // kmp_task_t *new_task);
5151 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5152 // kmp_task_t *new_task);
5153 RegionCodeGenTy RCG(CodeGen);
5154 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
5155 M, OMPRTL___kmpc_omp_task_begin_if0),
5156 TaskArgs,
5157 OMPBuilder.getOrCreateRuntimeFunction(
5158 M, OMPRTL___kmpc_omp_task_complete_if0),
5159 TaskArgs);
5160 RCG.setAction(Action);
5161 RCG(CGF);
5162 };
5163
5164 if (IfCond) {
5165 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5166 } else {
5167 RegionCodeGenTy ThenRCG(ThenCodeGen);
5168 ThenRCG(CGF);
5169 }
5170}
5171
5172void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5173 const OMPLoopDirective &D,
5174 llvm::Function *TaskFunction,
5175 QualType SharedsTy, Address Shareds,
5176 const Expr *IfCond,
5177 const OMPTaskDataTy &Data) {
5178 if (!CGF.HaveInsertPoint())
5179 return;
5180 TaskResultTy Result =
5181 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5182 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5183 // libcall.
5184 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5185 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5186 // sched, kmp_uint64 grainsize, void *task_dup);
5187 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5188 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5189 llvm::Value *IfVal;
5190 if (IfCond) {
5191 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5192 /*isSigned=*/true);
5193 } else {
5194 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5195 }
5196
5197 LValue LBLVal = CGF.EmitLValueForField(
5198 Result.TDBase,
5199 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5200 const auto *LBVar =
5201 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5202 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5203 LBLVal.getQuals(),
5204 /*IsInitializer=*/true);
5205 LValue UBLVal = CGF.EmitLValueForField(
5206 Result.TDBase,
5207 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5208 const auto *UBVar =
5209 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5210 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5211 UBLVal.getQuals(),
5212 /*IsInitializer=*/true);
5213 LValue StLVal = CGF.EmitLValueForField(
5214 Result.TDBase,
5215 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5216 const auto *StVar =
5217 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5218 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5219 StLVal.getQuals(),
5220 /*IsInitializer=*/true);
5221 // Store reductions address.
5222 LValue RedLVal = CGF.EmitLValueForField(
5223 Result.TDBase,
5224 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5225 if (Data.Reductions) {
5226 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5227 } else {
5228 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5229 CGF.getContext().VoidPtrTy);
5230 }
5231 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5232 llvm::Value *TaskArgs[] = {
5233 UpLoc,
5234 ThreadID,
5235 Result.NewTask,
5236 IfVal,
5237 LBLVal.getPointer(CGF),
5238 UBLVal.getPointer(CGF),
5239 CGF.EmitLoadOfScalar(StLVal, Loc),
5240 llvm::ConstantInt::getSigned(
5241 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5242 llvm::ConstantInt::getSigned(
5243 CGF.IntTy, Data.Schedule.getPointer()
5244 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5245 : NoSchedule),
5246 Data.Schedule.getPointer()
5247 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5248 /*isSigned=*/false)
5249 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5250 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5251 Result.TaskDupFn, CGF.VoidPtrTy)
5252 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5253 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5254 CGM.getModule(), OMPRTL___kmpc_taskloop),
5255 TaskArgs);
5256}
5257
5258/// Emit reduction operation for each element of array (required for
5259/// array sections) LHS op = RHS.
5260/// \param Type Type of array.
5261/// \param LHSVar Variable on the left side of the reduction operation
5262/// (references element of array in original variable).
5263/// \param RHSVar Variable on the right side of the reduction operation
5264/// (references element of array in original variable).
5265/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5266/// RHSVar.
5267static void EmitOMPAggregateReduction(
5268 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5269 const VarDecl *RHSVar,
5270 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5271 const Expr *, const Expr *)> &RedOpGen,
5272 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5273 const Expr *UpExpr = nullptr) {
5274 // Perform element-by-element initialization.
5275 QualType ElementTy;
5276 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5277 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5278
5279 // Drill down to the base element type on both arrays.
5280 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5281 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5282
5283 llvm::Value *RHSBegin = RHSAddr.getPointer();
5284 llvm::Value *LHSBegin = LHSAddr.getPointer();
5285 // Cast from pointer to array type to pointer to single element.
5286 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5287 // The basic structure here is a while-do loop.
5288 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5289 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5290 llvm::Value *IsEmpty =
5291 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5292 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5293
5294 // Enter the loop body, making that address the current address.
5295 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5296 CGF.EmitBlock(BodyBB);
5297
5298 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5299
5300 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5301 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5302 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5303 Address RHSElementCurrent =
5304 Address(RHSElementPHI,
5305 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5306
5307 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5308 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5309 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5310 Address LHSElementCurrent =
5311 Address(LHSElementPHI,
5312 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5313
5314 // Emit copy.
5315 CodeGenFunction::OMPPrivateScope Scope(CGF);
5316 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5317 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5318 Scope.Privatize();
5319 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5320 Scope.ForceCleanup();
5321
5322 // Shift the address forward by one element.
5323 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5324 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5325 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5326 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5327 // Check whether we've reached the end.
5328 llvm::Value *Done =
5329 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5330 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5331 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5332 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5333
5334 // Done.
5335 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5336}
5337
5338/// Emit reduction combiner. If the combiner is a simple expression emit it as
5339/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5340/// UDR combiner function.
5341static void emitReductionCombiner(CodeGenFunction &CGF,
5342 const Expr *ReductionOp) {
5343 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5344 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5345 if (const auto *DRE =
5346 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5347 if (const auto *DRD =
5348 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5349 std::pair<llvm::Function *, llvm::Function *> Reduction =
5350 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5351 RValue Func = RValue::get(Reduction.first);
5352 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5353 CGF.EmitIgnoredExpr(ReductionOp);
5354 return;
5355 }
5356 CGF.EmitIgnoredExpr(ReductionOp);
5357}
5358
5359llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5360 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5361 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5362 ArrayRef<const Expr *> ReductionOps) {
5363 ASTContext &C = CGM.getContext();
5364
5365 // void reduction_func(void *LHSArg, void *RHSArg);
5366 FunctionArgList Args;
5367 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5368 ImplicitParamDecl::Other);
5369 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5370 ImplicitParamDecl::Other);
5371 Args.push_back(&LHSArg);
5372 Args.push_back(&RHSArg);
5373 const auto &CGFI =
5374 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5375 std::string Name = getName({"omp", "reduction", "reduction_func"});
5376 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5377 llvm::GlobalValue::InternalLinkage, Name,
5378 &CGM.getModule());
5379 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5380 Fn->setDoesNotRecurse();
5381 CodeGenFunction CGF(CGM);
5382 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5383
5384 // Dst = (void*[n])(LHSArg);
5385 // Src = (void*[n])(RHSArg);
5386 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5387 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5388 ArgsType), CGF.getPointerAlign());
5389 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5390 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5391 ArgsType), CGF.getPointerAlign());
5392
5393 // ...
5394 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5395 // ...
5396 CodeGenFunction::OMPPrivateScope Scope(CGF);
5397 auto IPriv = Privates.begin();
5398 unsigned Idx = 0;
5399 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5400 const auto *RHSVar =
5401 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5402 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5403 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5404 });
5405 const auto *LHSVar =
5406 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5407 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5408 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5409 });
5410 QualType PrivTy = (*IPriv)->getType();
5411 if (PrivTy->isVariablyModifiedType()) {
5412 // Get array size and emit VLA type.
5413 ++Idx;
5414 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5415 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5416 const VariableArrayType *VLA =
5417 CGF.getContext().getAsVariableArrayType(PrivTy);
5418 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5419 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5420 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5421 CGF.EmitVariablyModifiedType(PrivTy);
5422 }
5423 }
5424 Scope.Privatize();
5425 IPriv = Privates.begin();
5426 auto ILHS = LHSExprs.begin();
5427 auto IRHS = RHSExprs.begin();
5428 for (const Expr *E : ReductionOps) {
5429 if ((*IPriv)->getType()->isArrayType()) {
5430 // Emit reduction for array section.
5431 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5432 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5433 EmitOMPAggregateReduction(
5434 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5435 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5436 emitReductionCombiner(CGF, E);
5437 });
5438 } else {
5439 // Emit reduction for array subscript or single variable.
5440 emitReductionCombiner(CGF, E);
5441 }
5442 ++IPriv;
5443 ++ILHS;
5444 ++IRHS;
5445 }
5446 Scope.ForceCleanup();
5447 CGF.FinishFunction();
5448 return Fn;
5449}
5450
5451void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5452 const Expr *ReductionOp,
5453 const Expr *PrivateRef,
5454 const DeclRefExpr *LHS,
5455 const DeclRefExpr *RHS) {
5456 if (PrivateRef->getType()->isArrayType()) {
5457 // Emit reduction for array section.
5458 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5459 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5460 EmitOMPAggregateReduction(
5461 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5462 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5463 emitReductionCombiner(CGF, ReductionOp);
5464 });
5465 } else {
5466 // Emit reduction for array subscript or single variable.
5467 emitReductionCombiner(CGF, ReductionOp);
5468 }
5469}
5470
5471void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5472 ArrayRef<const Expr *> Privates,
5473 ArrayRef<const Expr *> LHSExprs,
5474 ArrayRef<const Expr *> RHSExprs,
5475 ArrayRef<const Expr *> ReductionOps,
5476 ReductionOptionsTy Options) {
5477 if (!CGF.HaveInsertPoint())
5478 return;
5479
5480 bool WithNowait = Options.WithNowait;
5481 bool SimpleReduction = Options.SimpleReduction;
5482
5483 // Next code should be emitted for reduction:
5484 //
5485 // static kmp_critical_name lock = { 0 };
5486 //
5487 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5488 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5489 // ...
5490 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5491 // *(Type<n>-1*)rhs[<n>-1]);
5492 // }
5493 //
5494 // ...
5495 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5496 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5497 // RedList, reduce_func, &<lock>)) {
5498 // case 1:
5499 // ...
5500 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5501 // ...
5502 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5503 // break;
5504 // case 2:
5505 // ...
5506 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5507 // ...
5508 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5509 // break;
5510 // default:;
5511 // }
5512 //
5513 // if SimpleReduction is true, only the next code is generated:
5514 // ...
5515 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5516 // ...
5517
5518 ASTContext &C = CGM.getContext();
5519
5520 if (SimpleReduction) {
5521 CodeGenFunction::RunCleanupsScope Scope(CGF);
5522 auto IPriv = Privates.begin();
5523 auto ILHS = LHSExprs.begin();
5524 auto IRHS = RHSExprs.begin();
5525 for (const Expr *E : ReductionOps) {
5526 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5527 cast<DeclRefExpr>(*IRHS));
5528 ++IPriv;
5529 ++ILHS;
5530 ++IRHS;
5531 }
5532 return;
5533 }
5534
5535 // 1. Build a list of reduction variables.
5536 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5537 auto Size = RHSExprs.size();
5538 for (const Expr *E : Privates) {
5539 if (E->getType()->isVariablyModifiedType())
5540 // Reserve place for array size.
5541 ++Size;
5542 }
5543 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5544 QualType ReductionArrayTy =
5545 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5546 /*IndexTypeQuals=*/0);
5547 Address ReductionList =
5548 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5549 auto IPriv = Privates.begin();
5550 unsigned Idx = 0;
5551 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5552 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5553 CGF.Builder.CreateStore(
5554 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5555 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5556 Elem);
5557 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5558 // Store array size.
5559 ++Idx;
5560 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5561 llvm::Value *Size = CGF.Builder.CreateIntCast(
5562 CGF.getVLASize(
5563 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5564 .NumElts,
5565 CGF.SizeTy, /*isSigned=*/false);
5566 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5567 Elem);
5568 }
5569 }
5570
5571 // 2. Emit reduce_func().
5572 llvm::Function *ReductionFn = emitReductionFunction(
5573 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5574 LHSExprs, RHSExprs, ReductionOps);
5575
5576 // 3. Create static kmp_critical_name lock = { 0 };
5577 std::string Name = getName({"reduction"});
5578 llvm::Value *Lock = getCriticalRegionLock(Name);
5579
5580 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5581 // RedList, reduce_func, &<lock>);
5582 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5583 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5584 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5585 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5586 ReductionList.getPointer(), CGF.VoidPtrTy);
5587 llvm::Value *Args[] = {
5588 IdentTLoc, // ident_t *<loc>
5589 ThreadId, // i32 <gtid>
5590 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5591 ReductionArrayTySize, // size_type sizeof(RedList)
5592 RL, // void *RedList
5593 ReductionFn, // void (*) (void *, void *) <reduce_func>
5594 Lock // kmp_critical_name *&<lock>
5595 };
5596 llvm::Value *Res = CGF.EmitRuntimeCall(
5597 OMPBuilder.getOrCreateRuntimeFunction(
5598 CGM.getModule(),
5599 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5600 Args);
5601
5602 // 5. Build switch(res)
5603 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5604 llvm::SwitchInst *SwInst =
5605 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5606
5607 // 6. Build case 1:
5608 // ...
5609 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5610 // ...
5611 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5612 // break;
5613 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5614 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5615 CGF.EmitBlock(Case1BB);
5616
5617 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5618 llvm::Value *EndArgs[] = {
5619 IdentTLoc, // ident_t *<loc>
5620 ThreadId, // i32 <gtid>
5621 Lock // kmp_critical_name *&<lock>
5622 };
5623 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5624 CodeGenFunction &CGF, PrePostActionTy &Action) {
5625 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5626 auto IPriv = Privates.begin();
5627 auto ILHS = LHSExprs.begin();
5628 auto IRHS = RHSExprs.begin();
5629 for (const Expr *E : ReductionOps) {
5630 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5631 cast<DeclRefExpr>(*IRHS));
5632 ++IPriv;
5633 ++ILHS;
5634 ++IRHS;
5635 }
5636 };
5637 RegionCodeGenTy RCG(CodeGen);
5638 CommonActionTy Action(
5639 nullptr, llvm::None,
5640 OMPBuilder.getOrCreateRuntimeFunction(
5641 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5642 : OMPRTL___kmpc_end_reduce),
5643 EndArgs);
5644 RCG.setAction(Action);
5645 RCG(CGF);
5646
5647 CGF.EmitBranch(DefaultBB);
5648
5649 // 7. Build case 2:
5650 // ...
5651 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5652 // ...
5653 // break;
5654 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5655 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5656 CGF.EmitBlock(Case2BB);
5657
5658 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5659 CodeGenFunction &CGF, PrePostActionTy &Action) {
5660 auto ILHS = LHSExprs.begin();
5661 auto IRHS = RHSExprs.begin();
5662 auto IPriv = Privates.begin();
5663 for (const Expr *E : ReductionOps) {
5664 const Expr *XExpr = nullptr;
5665 const Expr *EExpr = nullptr;
5666 const Expr *UpExpr = nullptr;
5667 BinaryOperatorKind BO = BO_Comma;
5668 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5669 if (BO->getOpcode() == BO_Assign) {
5670 XExpr = BO->getLHS();
5671 UpExpr = BO->getRHS();
5672 }
5673 }
5674 // Try to emit update expression as a simple atomic.
5675 const Expr *RHSExpr = UpExpr;
5676 if (RHSExpr) {
5677 // Analyze RHS part of the whole expression.
5678 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5679 RHSExpr->IgnoreParenImpCasts())) {
5680 // If this is a conditional operator, analyze its condition for
5681 // min/max reduction operator.
5682 RHSExpr = ACO->getCond();
5683 }
5684 if (const auto *BORHS =
5685 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5686 EExpr = BORHS->getRHS();
5687 BO = BORHS->getOpcode();
5688 }
5689 }
5690 if (XExpr) {
5691 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5692 auto &&AtomicRedGen = [BO, VD,
5693 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5694 const Expr *EExpr, const Expr *UpExpr) {
5695 LValue X = CGF.EmitLValue(XExpr);
5696 RValue E;
5697 if (EExpr)
5698 E = CGF.EmitAnyExpr(EExpr);
5699 CGF.EmitOMPAtomicSimpleUpdateExpr(
5700 X, E, BO, /*IsXLHSInRHSPart=*/true,
5701 llvm::AtomicOrdering::Monotonic, Loc,
5702 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5703 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5704 PrivateScope.addPrivate(
5705 VD, [&CGF, VD, XRValue, Loc]() {
5706 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5707 CGF.emitOMPSimpleStore(
5708 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5709 VD->getType().getNonReferenceType(), Loc);
5710 return LHSTemp;
5711 });
5712 (void)PrivateScope.Privatize();
5713 return CGF.EmitAnyExpr(UpExpr);
5714 });
5715 };
5716 if ((*IPriv)->getType()->isArrayType()) {
5717 // Emit atomic reduction for array section.
5718 const auto *RHSVar =
5719 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5720 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5721 AtomicRedGen, XExpr, EExpr, UpExpr);
5722 } else {
5723 // Emit atomic reduction for array subscript or single variable.
5724 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5725 }
5726 } else {
5727 // Emit as a critical region.
5728 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5729 const Expr *, const Expr *) {
5730 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5731 std::string Name = RT.getName({"atomic_reduction"});
5732 RT.emitCriticalRegion(
5733 CGF, Name,
5734 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5735 Action.Enter(CGF);
5736 emitReductionCombiner(CGF, E);
5737 },
5738 Loc);
5739 };
5740 if ((*IPriv)->getType()->isArrayType()) {
5741 const auto *LHSVar =
5742 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5743 const auto *RHSVar =
5744 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5745 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5746 CritRedGen);
5747 } else {
5748 CritRedGen(CGF, nullptr, nullptr, nullptr);
5749 }
5750 }
5751 ++ILHS;
5752 ++IRHS;
5753 ++IPriv;
5754 }
5755 };
5756 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5757 if (!WithNowait) {
5758 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5759 llvm::Value *EndArgs[] = {
5760 IdentTLoc, // ident_t *<loc>
5761 ThreadId, // i32 <gtid>
5762 Lock // kmp_critical_name *&<lock>
5763 };
5764 CommonActionTy Action(nullptr, llvm::None,
5765 OMPBuilder.getOrCreateRuntimeFunction(
5766 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5767 EndArgs);
5768 AtomicRCG.setAction(Action);
5769 AtomicRCG(CGF);
5770 } else {
5771 AtomicRCG(CGF);
5772 }
5773
5774 CGF.EmitBranch(DefaultBB);
5775 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5776}
5777
5778/// Generates unique name for artificial threadprivate variables.
5779/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5780static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5781 const Expr *Ref) {
5782 SmallString<256> Buffer;
5783 llvm::raw_svector_ostream Out(Buffer);
5784 const clang::DeclRefExpr *DE;
5785 const VarDecl *D = ::getBaseDecl(Ref, DE);
5786 if (!D)
5787 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5788 D = D->getCanonicalDecl();
5789 std::string Name = CGM.getOpenMPRuntime().getName(
5790 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5791 Out << Prefix << Name << "_"
5792 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5793 return std::string(Out.str());
5794}
5795
5796