Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8082, column 9
2nd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/tools/clang/lib/CodeGen -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D CLANG_ROUND_TRIP_CC1_ARGS=ON -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-05-202135-9119-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/FileManager.h"
25#include "clang/Basic/OpenMPKinds.h"
26#include "clang/Basic/SourceManager.h"
27#include "clang/CodeGen/ConstantInitBuilder.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/SetOperations.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/Bitcode/BitcodeReader.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Value.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40#include <numeric>
41
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46namespace {
47/// Base class for handling code generation inside OpenMP regions.
48class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
49public:
50 /// Kinds of OpenMP regions used in codegen.
51 enum CGOpenMPRegionKind {
52 /// Region with outlined function for standalone 'parallel'
53 /// directive.
54 ParallelOutlinedRegion,
55 /// Region with outlined function for standalone 'task' directive.
56 TaskOutlinedRegion,
57 /// Region for constructs that do not require function outlining,
58 /// like 'for', 'sections', 'atomic' etc. directives.
59 InlinedRegion,
60 /// Region with outlined function for standalone 'target' directive.
61 TargetRegion,
62 };
63
64 CGOpenMPRegionInfo(const CapturedStmt &CS,
65 const CGOpenMPRegionKind RegionKind,
66 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
67 bool HasCancel)
68 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
69 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
70
71 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
72 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
73 bool HasCancel)
74 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
75 Kind(Kind), HasCancel(HasCancel) {}
76
77 /// Get a variable or parameter for storing global thread id
78 /// inside OpenMP construct.
79 virtual const VarDecl *getThreadIDVariable() const = 0;
80
81 /// Emit the captured statement body.
82 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
83
84 /// Get an LValue for the current ThreadID variable.
85 /// \return LValue for thread id variable. This LValue always has type int32*.
86 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
87
88 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
89
90 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
91
92 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
93
94 bool hasCancel() const { return HasCancel; }
95
96 static bool classof(const CGCapturedStmtInfo *Info) {
97 return Info->getKind() == CR_OpenMP;
98 }
99
100 ~CGOpenMPRegionInfo() override = default;
101
102protected:
103 CGOpenMPRegionKind RegionKind;
104 RegionCodeGenTy CodeGen;
105 OpenMPDirectiveKind Kind;
106 bool HasCancel;
107};
108
109/// API for captured statement code generation in OpenMP constructs.
110class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
111public:
112 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
113 const RegionCodeGenTy &CodeGen,
114 OpenMPDirectiveKind Kind, bool HasCancel,
115 StringRef HelperName)
116 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
117 HasCancel),
118 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
119 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 119, __PRETTY_FUNCTION__))
;
120 }
121
122 /// Get a variable or parameter for storing global thread id
123 /// inside OpenMP construct.
124 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
125
126 /// Get the name of the capture helper.
127 StringRef getHelperName() const override { return HelperName; }
128
129 static bool classof(const CGCapturedStmtInfo *Info) {
130 return CGOpenMPRegionInfo::classof(Info) &&
131 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
132 ParallelOutlinedRegion;
133 }
134
135private:
136 /// A variable or parameter storing global thread id for OpenMP
137 /// constructs.
138 const VarDecl *ThreadIDVar;
139 StringRef HelperName;
140};
141
142/// API for captured statement code generation in OpenMP constructs.
143class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
144public:
145 class UntiedTaskActionTy final : public PrePostActionTy {
146 bool Untied;
147 const VarDecl *PartIDVar;
148 const RegionCodeGenTy UntiedCodeGen;
149 llvm::SwitchInst *UntiedSwitch = nullptr;
150
151 public:
152 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
153 const RegionCodeGenTy &UntiedCodeGen)
154 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
155 void Enter(CodeGenFunction &CGF) override {
156 if (Untied) {
157 // Emit task switching point.
158 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
159 CGF.GetAddrOfLocalVar(PartIDVar),
160 PartIDVar->getType()->castAs<PointerType>());
161 llvm::Value *Res =
162 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
163 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
164 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
165 CGF.EmitBlock(DoneBB);
166 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
167 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
168 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
169 CGF.Builder.GetInsertBlock());
170 emitUntiedSwitch(CGF);
171 }
172 }
173 void emitUntiedSwitch(CodeGenFunction &CGF) const {
174 if (Untied) {
175 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
176 CGF.GetAddrOfLocalVar(PartIDVar),
177 PartIDVar->getType()->castAs<PointerType>());
178 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
179 PartIdLVal);
180 UntiedCodeGen(CGF);
181 CodeGenFunction::JumpDest CurPoint =
182 CGF.getJumpDestInCurrentScope(".untied.next.");
183 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
184 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
185 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
186 CGF.Builder.GetInsertBlock());
187 CGF.EmitBranchThroughCleanup(CurPoint);
188 CGF.EmitBlock(CurPoint.getBlock());
189 }
190 }
191 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
192 };
193 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
194 const VarDecl *ThreadIDVar,
195 const RegionCodeGenTy &CodeGen,
196 OpenMPDirectiveKind Kind, bool HasCancel,
197 const UntiedTaskActionTy &Action)
198 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
199 ThreadIDVar(ThreadIDVar), Action(Action) {
200 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 200, __PRETTY_FUNCTION__))
;
201 }
202
203 /// Get a variable or parameter for storing global thread id
204 /// inside OpenMP construct.
205 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
206
207 /// Get an LValue for the current ThreadID variable.
208 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
209
210 /// Get the name of the capture helper.
211 StringRef getHelperName() const override { return ".omp_outlined."; }
212
213 void emitUntiedSwitch(CodeGenFunction &CGF) override {
214 Action.emitUntiedSwitch(CGF);
215 }
216
217 static bool classof(const CGCapturedStmtInfo *Info) {
218 return CGOpenMPRegionInfo::classof(Info) &&
219 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
220 TaskOutlinedRegion;
221 }
222
223private:
224 /// A variable or parameter storing global thread id for OpenMP
225 /// constructs.
226 const VarDecl *ThreadIDVar;
227 /// Action for emitting code for untied tasks.
228 const UntiedTaskActionTy &Action;
229};
230
231/// API for inlined captured statement code generation in OpenMP
232/// constructs.
233class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
234public:
235 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
236 const RegionCodeGenTy &CodeGen,
237 OpenMPDirectiveKind Kind, bool HasCancel)
238 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
239 OldCSI(OldCSI),
240 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
241
242 // Retrieve the value of the context parameter.
243 llvm::Value *getContextValue() const override {
244 if (OuterRegionInfo)
245 return OuterRegionInfo->getContextValue();
246 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 246)
;
247 }
248
249 void setContextValue(llvm::Value *V) override {
250 if (OuterRegionInfo) {
251 OuterRegionInfo->setContextValue(V);
252 return;
253 }
254 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 254)
;
255 }
256
257 /// Lookup the captured field decl for a variable.
258 const FieldDecl *lookup(const VarDecl *VD) const override {
259 if (OuterRegionInfo)
260 return OuterRegionInfo->lookup(VD);
261 // If there is no outer outlined region,no need to lookup in a list of
262 // captured variables, we can use the original one.
263 return nullptr;
264 }
265
266 FieldDecl *getThisFieldDecl() const override {
267 if (OuterRegionInfo)
268 return OuterRegionInfo->getThisFieldDecl();
269 return nullptr;
270 }
271
272 /// Get a variable or parameter for storing global thread id
273 /// inside OpenMP construct.
274 const VarDecl *getThreadIDVariable() const override {
275 if (OuterRegionInfo)
276 return OuterRegionInfo->getThreadIDVariable();
277 return nullptr;
278 }
279
280 /// Get an LValue for the current ThreadID variable.
281 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
284 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 284)
;
285 }
286
287 /// Get the name of the capture helper.
288 StringRef getHelperName() const override {
289 if (auto *OuterRegionInfo = getOldCSI())
290 return OuterRegionInfo->getHelperName();
291 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 291)
;
292 }
293
294 void emitUntiedSwitch(CodeGenFunction &CGF) override {
295 if (OuterRegionInfo)
296 OuterRegionInfo->emitUntiedSwitch(CGF);
297 }
298
299 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
300
301 static bool classof(const CGCapturedStmtInfo *Info) {
302 return CGOpenMPRegionInfo::classof(Info) &&
303 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
304 }
305
306 ~CGOpenMPInlinedRegionInfo() override = default;
307
308private:
309 /// CodeGen info about outer OpenMP region.
310 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
311 CGOpenMPRegionInfo *OuterRegionInfo;
312};
313
314/// API for captured statement code generation in OpenMP target
315/// constructs. For this captures, implicit parameters are used instead of the
316/// captured fields. The name of the target region has to be unique in a given
317/// application so it is provided by the client, because only the client has
318/// the information to generate that.
319class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
320public:
321 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
322 const RegionCodeGenTy &CodeGen, StringRef HelperName)
323 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
324 /*HasCancel=*/false),
325 HelperName(HelperName) {}
326
327 /// This is unused for target regions because each starts executing
328 /// with a single thread.
329 const VarDecl *getThreadIDVariable() const override { return nullptr; }
330
331 /// Get the name of the capture helper.
332 StringRef getHelperName() const override { return HelperName; }
333
334 static bool classof(const CGCapturedStmtInfo *Info) {
335 return CGOpenMPRegionInfo::classof(Info) &&
336 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
337 }
338
339private:
340 StringRef HelperName;
341};
342
343static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
344 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 344)
;
345}
346/// API for generation of expressions captured in a innermost OpenMP
347/// region.
348class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
349public:
350 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
351 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
352 OMPD_unknown,
353 /*HasCancel=*/false),
354 PrivScope(CGF) {
355 // Make sure the globals captured in the provided statement are local by
356 // using the privatization logic. We assume the same variable is not
357 // captured more than once.
358 for (const auto &C : CS.captures()) {
359 if (!C.capturesVariable() && !C.capturesVariableByCopy())
360 continue;
361
362 const VarDecl *VD = C.getCapturedVar();
363 if (VD->isLocalVarDeclOrParm())
364 continue;
365
366 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
367 /*RefersToEnclosingVariableOrCapture=*/false,
368 VD->getType().getNonReferenceType(), VK_LValue,
369 C.getLocation());
370 PrivScope.addPrivate(
371 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
372 }
373 (void)PrivScope.Privatize();
374 }
375
376 /// Lookup the captured field decl for a variable.
377 const FieldDecl *lookup(const VarDecl *VD) const override {
378 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
379 return FD;
380 return nullptr;
381 }
382
383 /// Emit the captured statement body.
384 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
385 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 385)
;
386 }
387
388 /// Get a variable or parameter for storing global thread id
389 /// inside OpenMP construct.
390 const VarDecl *getThreadIDVariable() const override {
391 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 391)
;
392 }
393
394 /// Get the name of the capture helper.
395 StringRef getHelperName() const override {
396 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 396)
;
397 }
398
399 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
400
401private:
402 /// Private scope to capture global variables.
403 CodeGenFunction::OMPPrivateScope PrivScope;
404};
405
406/// RAII for emitting code of OpenMP constructs.
407class InlinedOpenMPRegionRAII {
408 CodeGenFunction &CGF;
409 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
410 FieldDecl *LambdaThisCaptureField = nullptr;
411 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
412 bool NoInheritance = false;
413
414public:
415 /// Constructs region for combined constructs.
416 /// \param CodeGen Code generation sequence for combined directives. Includes
417 /// a list of functions used for code generation of implicitly inlined
418 /// regions.
419 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
420 OpenMPDirectiveKind Kind, bool HasCancel,
421 bool NoInheritance = true)
422 : CGF(CGF), NoInheritance(NoInheritance) {
423 // Start emission for the construct.
424 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
425 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
426 if (NoInheritance) {
427 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
428 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
429 CGF.LambdaThisCaptureField = nullptr;
430 BlockInfo = CGF.BlockInfo;
431 CGF.BlockInfo = nullptr;
432 }
433 }
434
435 ~InlinedOpenMPRegionRAII() {
436 // Restore original CapturedStmtInfo only if we're done with code emission.
437 auto *OldCSI =
438 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
439 delete CGF.CapturedStmtInfo;
440 CGF.CapturedStmtInfo = OldCSI;
441 if (NoInheritance) {
442 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
443 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
444 CGF.BlockInfo = BlockInfo;
445 }
446 }
447};
448
449/// Values for bit flags used in the ident_t to describe the fields.
450/// All enumeric elements are named and described in accordance with the code
451/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
452enum OpenMPLocationFlags : unsigned {
453 /// Use trampoline for internal microtask.
454 OMP_IDENT_IMD = 0x01,
455 /// Use c-style ident structure.
456 OMP_IDENT_KMPC = 0x02,
457 /// Atomic reduction option for kmpc_reduce.
458 OMP_ATOMIC_REDUCE = 0x10,
459 /// Explicit 'barrier' directive.
460 OMP_IDENT_BARRIER_EXPL = 0x20,
461 /// Implicit barrier in code.
462 OMP_IDENT_BARRIER_IMPL = 0x40,
463 /// Implicit barrier in 'for' directive.
464 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
465 /// Implicit barrier in 'sections' directive.
466 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
467 /// Implicit barrier in 'single' directive.
468 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
469 /// Call of __kmp_for_static_init for static loop.
470 OMP_IDENT_WORK_LOOP = 0x200,
471 /// Call of __kmp_for_static_init for sections.
472 OMP_IDENT_WORK_SECTIONS = 0x400,
473 /// Call of __kmp_for_static_init for distribute.
474 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
475 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
476};
477
478namespace {
479LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
480/// Values for bit flags for marking which requires clauses have been used.
481enum OpenMPOffloadingRequiresDirFlags : int64_t {
482 /// flag undefined.
483 OMP_REQ_UNDEFINED = 0x000,
484 /// no requires clause present.
485 OMP_REQ_NONE = 0x001,
486 /// reverse_offload clause.
487 OMP_REQ_REVERSE_OFFLOAD = 0x002,
488 /// unified_address clause.
489 OMP_REQ_UNIFIED_ADDRESS = 0x004,
490 /// unified_shared_memory clause.
491 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
492 /// dynamic_allocators clause.
493 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
494 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
495};
496
497enum OpenMPOffloadingReservedDeviceIDs {
498 /// Device ID if the device was not defined, runtime should get it
499 /// from environment variables in the spec.
500 OMP_DEVICEID_UNDEF = -1,
501};
502} // anonymous namespace
503
504/// Describes ident structure that describes a source location.
505/// All descriptions are taken from
506/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
507/// Original structure:
508/// typedef struct ident {
509/// kmp_int32 reserved_1; /**< might be used in Fortran;
510/// see above */
511/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
512/// KMP_IDENT_KMPC identifies this union
513/// member */
514/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
515/// see above */
516///#if USE_ITT_BUILD
517/// /* but currently used for storing
518/// region-specific ITT */
519/// /* contextual information. */
520///#endif /* USE_ITT_BUILD */
521/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
522/// C++ */
523/// char const *psource; /**< String describing the source location.
524/// The string is composed of semi-colon separated
525// fields which describe the source file,
526/// the function and a pair of line numbers that
527/// delimit the construct.
528/// */
529/// } ident_t;
530enum IdentFieldIndex {
531 /// might be used in Fortran
532 IdentField_Reserved_1,
533 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
534 IdentField_Flags,
535 /// Not really used in Fortran any more
536 IdentField_Reserved_2,
537 /// Source[4] in Fortran, do not use for C++
538 IdentField_Reserved_3,
539 /// String describing the source location. The string is composed of
540 /// semi-colon separated fields which describe the source file, the function
541 /// and a pair of line numbers that delimit the construct.
542 IdentField_PSource
543};
544
545/// Schedule types for 'omp for' loops (these enumerators are taken from
546/// the enum sched_type in kmp.h).
547enum OpenMPSchedType {
548 /// Lower bound for default (unordered) versions.
549 OMP_sch_lower = 32,
550 OMP_sch_static_chunked = 33,
551 OMP_sch_static = 34,
552 OMP_sch_dynamic_chunked = 35,
553 OMP_sch_guided_chunked = 36,
554 OMP_sch_runtime = 37,
555 OMP_sch_auto = 38,
556 /// static with chunk adjustment (e.g., simd)
557 OMP_sch_static_balanced_chunked = 45,
558 /// Lower bound for 'ordered' versions.
559 OMP_ord_lower = 64,
560 OMP_ord_static_chunked = 65,
561 OMP_ord_static = 66,
562 OMP_ord_dynamic_chunked = 67,
563 OMP_ord_guided_chunked = 68,
564 OMP_ord_runtime = 69,
565 OMP_ord_auto = 70,
566 OMP_sch_default = OMP_sch_static,
567 /// dist_schedule types
568 OMP_dist_sch_static_chunked = 91,
569 OMP_dist_sch_static = 92,
570 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
571 /// Set if the monotonic schedule modifier was present.
572 OMP_sch_modifier_monotonic = (1 << 29),
573 /// Set if the nonmonotonic schedule modifier was present.
574 OMP_sch_modifier_nonmonotonic = (1 << 30),
575};
576
577/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
578/// region.
579class CleanupTy final : public EHScopeStack::Cleanup {
580 PrePostActionTy *Action;
581
582public:
583 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
584 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
585 if (!CGF.HaveInsertPoint())
586 return;
587 Action->Exit(CGF);
588 }
589};
590
591} // anonymous namespace
592
593void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
594 CodeGenFunction::RunCleanupsScope Scope(CGF);
595 if (PrePostAction) {
596 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
597 Callback(CodeGen, CGF, *PrePostAction);
598 } else {
599 PrePostActionTy Action;
600 Callback(CodeGen, CGF, Action);
601 }
602}
603
604/// Check if the combiner is a call to UDR combiner and if it is so return the
605/// UDR decl used for reduction.
606static const OMPDeclareReductionDecl *
607getReductionInit(const Expr *ReductionOp) {
608 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
609 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
610 if (const auto *DRE =
611 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
612 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
613 return DRD;
614 return nullptr;
615}
616
617static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
618 const OMPDeclareReductionDecl *DRD,
619 const Expr *InitOp,
620 Address Private, Address Original,
621 QualType Ty) {
622 if (DRD->getInitializer()) {
623 std::pair<llvm::Function *, llvm::Function *> Reduction =
624 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
625 const auto *CE = cast<CallExpr>(InitOp);
626 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
627 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
628 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
629 const auto *LHSDRE =
630 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
631 const auto *RHSDRE =
632 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
633 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
634 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
635 [=]() { return Private; });
636 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
637 [=]() { return Original; });
638 (void)PrivateScope.Privatize();
639 RValue Func = RValue::get(Reduction.second);
640 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
641 CGF.EmitIgnoredExpr(InitOp);
642 } else {
643 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
644 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
645 auto *GV = new llvm::GlobalVariable(
646 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
647 llvm::GlobalValue::PrivateLinkage, Init, Name);
648 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
649 RValue InitRVal;
650 switch (CGF.getEvaluationKind(Ty)) {
651 case TEK_Scalar:
652 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
653 break;
654 case TEK_Complex:
655 InitRVal =
656 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
657 break;
658 case TEK_Aggregate: {
659 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
660 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
661 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
662 /*IsInitializer=*/false);
663 return;
664 }
665 }
666 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
667 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
668 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
669 /*IsInitializer=*/false);
670 }
671}
672
673/// Emit initialization of arrays of complex types.
674/// \param DestAddr Address of the array.
675/// \param Type Type of array.
676/// \param Init Initial expression of array.
677/// \param SrcAddr Address of the original array.
678static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
679 QualType Type, bool EmitDeclareReductionInit,
680 const Expr *Init,
681 const OMPDeclareReductionDecl *DRD,
682 Address SrcAddr = Address::invalid()) {
683 // Perform element-by-element initialization.
684 QualType ElementTy;
685
686 // Drill down to the base element type on both arrays.
687 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
688 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
689 DestAddr =
690 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
691 if (DRD)
692 SrcAddr =
693 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
694
695 llvm::Value *SrcBegin = nullptr;
696 if (DRD)
697 SrcBegin = SrcAddr.getPointer();
698 llvm::Value *DestBegin = DestAddr.getPointer();
699 // Cast from pointer to array type to pointer to single element.
700 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
701 // The basic structure here is a while-do loop.
702 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
703 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
704 llvm::Value *IsEmpty =
705 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
706 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
707
708 // Enter the loop body, making that address the current address.
709 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
710 CGF.EmitBlock(BodyBB);
711
712 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
713
714 llvm::PHINode *SrcElementPHI = nullptr;
715 Address SrcElementCurrent = Address::invalid();
716 if (DRD) {
717 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
718 "omp.arraycpy.srcElementPast");
719 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
720 SrcElementCurrent =
721 Address(SrcElementPHI,
722 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
723 }
724 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
725 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
726 DestElementPHI->addIncoming(DestBegin, EntryBB);
727 Address DestElementCurrent =
728 Address(DestElementPHI,
729 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
730
731 // Emit copy.
732 {
733 CodeGenFunction::RunCleanupsScope InitScope(CGF);
734 if (EmitDeclareReductionInit) {
735 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
736 SrcElementCurrent, ElementTy);
737 } else
738 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
739 /*IsInitializer=*/false);
740 }
741
742 if (DRD) {
743 // Shift the address forward by one element.
744 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
745 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
746 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
747 }
748
749 // Shift the address forward by one element.
750 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
751 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
752 // Check whether we've reached the end.
753 llvm::Value *Done =
754 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
755 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
756 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
757
758 // Done.
759 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
760}
761
762LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
763 return CGF.EmitOMPSharedLValue(E);
764}
765
766LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
767 const Expr *E) {
768 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
769 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
770 return LValue();
771}
772
773void ReductionCodeGen::emitAggregateInitialization(
774 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
775 const OMPDeclareReductionDecl *DRD) {
776 // Emit VarDecl with copy init for arrays.
777 // Get the address of the original variable captured in current
778 // captured region.
779 const auto *PrivateVD =
780 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
781 bool EmitDeclareReductionInit =
782 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
783 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
784 EmitDeclareReductionInit,
785 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
786 : PrivateVD->getInit(),
787 DRD, SharedLVal.getAddress(CGF));
788}
789
790ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
791 ArrayRef<const Expr *> Origs,
792 ArrayRef<const Expr *> Privates,
793 ArrayRef<const Expr *> ReductionOps) {
794 ClausesData.reserve(Shareds.size());
795 SharedAddresses.reserve(Shareds.size());
796 Sizes.reserve(Shareds.size());
797 BaseDecls.reserve(Shareds.size());
798 const auto *IOrig = Origs.begin();
799 const auto *IPriv = Privates.begin();
800 const auto *IRed = ReductionOps.begin();
801 for (const Expr *Ref : Shareds) {
802 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
803 std::advance(IOrig, 1);
804 std::advance(IPriv, 1);
805 std::advance(IRed, 1);
806 }
807}
808
809void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
810 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 811, __PRETTY_FUNCTION__))
811 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 811, __PRETTY_FUNCTION__))
;
812 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
813 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
814 SharedAddresses.emplace_back(First, Second);
815 if (ClausesData[N].Shared == ClausesData[N].Ref) {
816 OrigAddresses.emplace_back(First, Second);
817 } else {
818 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
819 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
820 OrigAddresses.emplace_back(First, Second);
821 }
822}
823
824void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
825 const auto *PrivateVD =
826 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
827 QualType PrivateType = PrivateVD->getType();
828 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
829 if (!PrivateType->isVariablyModifiedType()) {
830 Sizes.emplace_back(
831 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
832 nullptr);
833 return;
834 }
835 llvm::Value *Size;
836 llvm::Value *SizeInChars;
837 auto *ElemType =
838 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType())
839 ->getElementType();
840 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
841 if (AsArraySection) {
842 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
843 OrigAddresses[N].first.getPointer(CGF));
844 Size = CGF.Builder.CreateNUWAdd(
845 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
846 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
847 } else {
848 SizeInChars =
849 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
850 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
851 }
852 Sizes.emplace_back(SizeInChars, Size);
853 CodeGenFunction::OpaqueValueMapping OpaqueMap(
854 CGF,
855 cast<OpaqueValueExpr>(
856 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
857 RValue::get(Size));
858 CGF.EmitVariablyModifiedType(PrivateType);
859}
860
861void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
862 llvm::Value *Size) {
863 const auto *PrivateVD =
864 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
865 QualType PrivateType = PrivateVD->getType();
866 if (!PrivateType->isVariablyModifiedType()) {
867 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 869, __PRETTY_FUNCTION__))
868 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 869, __PRETTY_FUNCTION__))
869 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 869, __PRETTY_FUNCTION__))
;
870 return;
871 }
872 CodeGenFunction::OpaqueValueMapping OpaqueMap(
873 CGF,
874 cast<OpaqueValueExpr>(
875 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
876 RValue::get(Size));
877 CGF.EmitVariablyModifiedType(PrivateType);
878}
879
880void ReductionCodeGen::emitInitialization(
881 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
882 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
883 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 883, __PRETTY_FUNCTION__))
;
884 const auto *PrivateVD =
885 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
886 const OMPDeclareReductionDecl *DRD =
887 getReductionInit(ClausesData[N].ReductionOp);
888 QualType PrivateType = PrivateVD->getType();
889 PrivateAddr = CGF.Builder.CreateElementBitCast(
890 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
891 QualType SharedType = SharedAddresses[N].first.getType();
892 SharedLVal = CGF.MakeAddrLValue(
893 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
894 CGF.ConvertTypeForMem(SharedType)),
895 SharedType, SharedAddresses[N].first.getBaseInfo(),
896 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
897 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
898 if (DRD && DRD->getInitializer())
899 (void)DefaultInit(CGF);
900 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
901 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
902 (void)DefaultInit(CGF);
903 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
904 PrivateAddr, SharedLVal.getAddress(CGF),
905 SharedLVal.getType());
906 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
907 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
908 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
909 PrivateVD->getType().getQualifiers(),
910 /*IsInitializer=*/false);
911 }
912}
913
914bool ReductionCodeGen::needCleanups(unsigned N) {
915 const auto *PrivateVD =
916 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
917 QualType PrivateType = PrivateVD->getType();
918 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
919 return DTorKind != QualType::DK_none;
920}
921
922void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
923 Address PrivateAddr) {
924 const auto *PrivateVD =
925 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
926 QualType PrivateType = PrivateVD->getType();
927 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
928 if (needCleanups(N)) {
929 PrivateAddr = CGF.Builder.CreateElementBitCast(
930 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
931 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
932 }
933}
934
935static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
936 LValue BaseLV) {
937 BaseTy = BaseTy.getNonReferenceType();
938 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
939 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
940 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
941 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
942 } else {
943 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
944 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
945 }
946 BaseTy = BaseTy->getPointeeType();
947 }
948 return CGF.MakeAddrLValue(
949 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
950 CGF.ConvertTypeForMem(ElTy)),
951 BaseLV.getType(), BaseLV.getBaseInfo(),
952 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
953}
954
955static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
956 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
957 llvm::Value *Addr) {
958 Address Tmp = Address::invalid();
959 Address TopTmp = Address::invalid();
960 Address MostTopTmp = Address::invalid();
961 BaseTy = BaseTy.getNonReferenceType();
962 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
963 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
964 Tmp = CGF.CreateMemTemp(BaseTy);
965 if (TopTmp.isValid())
966 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
967 else
968 MostTopTmp = Tmp;
969 TopTmp = Tmp;
970 BaseTy = BaseTy->getPointeeType();
971 }
972 llvm::Type *Ty = BaseLVType;
973 if (Tmp.isValid())
974 Ty = Tmp.getElementType();
975 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
976 if (Tmp.isValid()) {
977 CGF.Builder.CreateStore(Addr, Tmp);
978 return MostTopTmp;
979 }
980 return Address(Addr, BaseLVAlignment);
981}
982
983static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
984 const VarDecl *OrigVD = nullptr;
985 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
986 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
987 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
988 Base = TempOASE->getBase()->IgnoreParenImpCasts();
989 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
990 Base = TempASE->getBase()->IgnoreParenImpCasts();
991 DE = cast<DeclRefExpr>(Base);
992 OrigVD = cast<VarDecl>(DE->getDecl());
993 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
994 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
995 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
996 Base = TempASE->getBase()->IgnoreParenImpCasts();
997 DE = cast<DeclRefExpr>(Base);
998 OrigVD = cast<VarDecl>(DE->getDecl());
999 }
1000 return OrigVD;
1001}
1002
1003Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1004 Address PrivateAddr) {
1005 const DeclRefExpr *DE;
1006 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1007 BaseDecls.emplace_back(OrigVD);
1008 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1009 LValue BaseLValue =
1010 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1011 OriginalBaseLValue);
1012 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1013 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1014 llvm::Value *PrivatePointer =
1015 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1016 PrivateAddr.getPointer(),
1017 SharedAddresses[N].first.getAddress(CGF).getType());
1018 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1019 return castToBase(CGF, OrigVD->getType(),
1020 SharedAddresses[N].first.getType(),
1021 OriginalBaseLValue.getAddress(CGF).getType(),
1022 OriginalBaseLValue.getAlignment(), Ptr);
1023 }
1024 BaseDecls.emplace_back(
1025 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1026 return PrivateAddr;
1027}
1028
1029bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1030 const OMPDeclareReductionDecl *DRD =
1031 getReductionInit(ClausesData[N].ReductionOp);
1032 return DRD && DRD->getInitializer();
1033}
1034
1035LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1036 return CGF.EmitLoadOfPointerLValue(
1037 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1038 getThreadIDVariable()->getType()->castAs<PointerType>());
1039}
1040
1041void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1042 if (!CGF.HaveInsertPoint())
1043 return;
1044 // 1.2.2 OpenMP Language Terminology
1045 // Structured block - An executable statement with a single entry at the
1046 // top and a single exit at the bottom.
1047 // The point of exit cannot be a branch out of the structured block.
1048 // longjmp() and throw() must not violate the entry/exit criteria.
1049 CGF.EHStack.pushTerminate();
1050 if (S)
1051 CGF.incrementProfileCounter(S);
1052 CodeGen(CGF);
1053 CGF.EHStack.popTerminate();
1054}
1055
1056LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1057 CodeGenFunction &CGF) {
1058 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1059 getThreadIDVariable()->getType(),
1060 AlignmentSource::Decl);
1061}
1062
1063static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1064 QualType FieldTy) {
1065 auto *Field = FieldDecl::Create(
1066 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1067 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1068 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1069 Field->setAccess(AS_public);
1070 DC->addDecl(Field);
1071 return Field;
1072}
1073
1074CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1075 StringRef Separator)
1076 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1077 OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
1078 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1079
1080 // Initialize Types used in OpenMPIRBuilder from OMPKinds.def
1081 OMPBuilder.initialize();
1082 loadOffloadInfoMetadata();
1083}
1084
1085void CGOpenMPRuntime::clear() {
1086 InternalVars.clear();
1087 // Clean non-target variable declarations possibly used only in debug info.
1088 for (const auto &Data : EmittedNonTargetVariables) {
1089 if (!Data.getValue().pointsToAliveValue())
1090 continue;
1091 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1092 if (!GV)
1093 continue;
1094 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1095 continue;
1096 GV->eraseFromParent();
1097 }
1098}
1099
1100std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1101 SmallString<128> Buffer;
1102 llvm::raw_svector_ostream OS(Buffer);
1103 StringRef Sep = FirstSeparator;
1104 for (StringRef Part : Parts) {
1105 OS << Sep << Part;
1106 Sep = Separator;
1107 }
1108 return std::string(OS.str());
1109}
1110
1111static llvm::Function *
1112emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1113 const Expr *CombinerInitializer, const VarDecl *In,
1114 const VarDecl *Out, bool IsCombiner) {
1115 // void .omp_combiner.(Ty *in, Ty *out);
1116 ASTContext &C = CGM.getContext();
1117 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1118 FunctionArgList Args;
1119 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1120 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1121 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1122 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1123 Args.push_back(&OmpOutParm);
1124 Args.push_back(&OmpInParm);
1125 const CGFunctionInfo &FnInfo =
1126 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1127 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1128 std::string Name = CGM.getOpenMPRuntime().getName(
1129 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1130 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1131 Name, &CGM.getModule());
1132 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1133 if (CGM.getLangOpts().Optimize) {
1134 Fn->removeFnAttr(llvm::Attribute::NoInline);
1135 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1136 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1137 }
1138 CodeGenFunction CGF(CGM);
1139 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1140 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1141 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1142 Out->getLocation());
1143 CodeGenFunction::OMPPrivateScope Scope(CGF);
1144 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1145 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1146 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1147 .getAddress(CGF);
1148 });
1149 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1150 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1151 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1152 .getAddress(CGF);
1153 });
1154 (void)Scope.Privatize();
1155 if (!IsCombiner && Out->hasInit() &&
1156 !CGF.isTrivialInitializer(Out->getInit())) {
1157 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1158 Out->getType().getQualifiers(),
1159 /*IsInitializer=*/true);
1160 }
1161 if (CombinerInitializer)
1162 CGF.EmitIgnoredExpr(CombinerInitializer);
1163 Scope.ForceCleanup();
1164 CGF.FinishFunction();
1165 return Fn;
1166}
1167
1168void CGOpenMPRuntime::emitUserDefinedReduction(
1169 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1170 if (UDRMap.count(D) > 0)
1171 return;
1172 llvm::Function *Combiner = emitCombinerOrInitializer(
1173 CGM, D->getType(), D->getCombiner(),
1174 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1175 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1176 /*IsCombiner=*/true);
1177 llvm::Function *Initializer = nullptr;
1178 if (const Expr *Init = D->getInitializer()) {
1179 Initializer = emitCombinerOrInitializer(
1180 CGM, D->getType(),
1181 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1182 : nullptr,
1183 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1184 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1185 /*IsCombiner=*/false);
1186 }
1187 UDRMap.try_emplace(D, Combiner, Initializer);
1188 if (CGF) {
1189 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1190 Decls.second.push_back(D);
1191 }
1192}
1193
1194std::pair<llvm::Function *, llvm::Function *>
1195CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1196 auto I = UDRMap.find(D);
1197 if (I != UDRMap.end())
1198 return I->second;
1199 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1200 return UDRMap.lookup(D);
1201}
1202
1203namespace {
1204// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1205// Builder if one is present.
1206struct PushAndPopStackRAII {
1207 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1208 bool HasCancel)
1209 : OMPBuilder(OMPBuilder) {
1210 if (!OMPBuilder)
1211 return;
1212
1213 // The following callback is the crucial part of clangs cleanup process.
1214 //
1215 // NOTE:
1216 // Once the OpenMPIRBuilder is used to create parallel regions (and
1217 // similar), the cancellation destination (Dest below) is determined via
1218 // IP. That means if we have variables to finalize we split the block at IP,
1219 // use the new block (=BB) as destination to build a JumpDest (via
1220 // getJumpDestInCurrentScope(BB)) which then is fed to
1221 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1222 // to push & pop an FinalizationInfo object.
1223 // The FiniCB will still be needed but at the point where the
1224 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1225 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1226 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1227, __PRETTY_FUNCTION__))
1227 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1227, __PRETTY_FUNCTION__))
;
1228 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1229 CGF.Builder.restoreIP(IP);
1230 CodeGenFunction::JumpDest Dest =
1231 CGF.getOMPCancelDestination(OMPD_parallel);
1232 CGF.EmitBranchThroughCleanup(Dest);
1233 };
1234
1235 // TODO: Remove this once we emit parallel regions through the
1236 // OpenMPIRBuilder as it can do this setup internally.
1237 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1238 {FiniCB, OMPD_parallel, HasCancel});
1239 OMPBuilder->pushFinalizationCB(std::move(FI));
1240 }
1241 ~PushAndPopStackRAII() {
1242 if (OMPBuilder)
1243 OMPBuilder->popFinalizationCB();
1244 }
1245 llvm::OpenMPIRBuilder *OMPBuilder;
1246};
1247} // namespace
1248
1249static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1250 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1251 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1252 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1253 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1254, __PRETTY_FUNCTION__))
1254 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1254, __PRETTY_FUNCTION__))
;
1255 CodeGenFunction CGF(CGM, true);
1256 bool HasCancel = false;
1257 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1258 HasCancel = OPD->hasCancel();
1259 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1260 HasCancel = OPD->hasCancel();
1261 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1262 HasCancel = OPSD->hasCancel();
1263 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1264 HasCancel = OPFD->hasCancel();
1265 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1266 HasCancel = OPFD->hasCancel();
1267 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1268 HasCancel = OPFD->hasCancel();
1269 else if (const auto *OPFD =
1270 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1271 HasCancel = OPFD->hasCancel();
1272 else if (const auto *OPFD =
1273 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1274 HasCancel = OPFD->hasCancel();
1275
1276 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1277 // parallel region to make cancellation barriers work properly.
1278 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1279 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel);
1280 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1281 HasCancel, OutlinedHelperName);
1282 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1283 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1284}
1285
1286llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1287 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1288 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1289 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1290 return emitParallelOrTeamsOutlinedFunction(
1291 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1292}
1293
1294llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1295 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1296 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1297 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1298 return emitParallelOrTeamsOutlinedFunction(
1299 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1300}
1301
1302llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1303 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1304 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1305 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1306 bool Tied, unsigned &NumberOfParts) {
1307 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1308 PrePostActionTy &) {
1309 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1310 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1311 llvm::Value *TaskArgs[] = {
1312 UpLoc, ThreadID,
1313 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1314 TaskTVar->getType()->castAs<PointerType>())
1315 .getPointer(CGF)};
1316 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1317 CGM.getModule(), OMPRTL___kmpc_omp_task),
1318 TaskArgs);
1319 };
1320 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1321 UntiedCodeGen);
1322 CodeGen.setAction(Action);
1323 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1324, __PRETTY_FUNCTION__))
1324 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1324, __PRETTY_FUNCTION__))
;
1325 const OpenMPDirectiveKind Region =
1326 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1327 : OMPD_task;
1328 const CapturedStmt *CS = D.getCapturedStmt(Region);
1329 bool HasCancel = false;
1330 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1331 HasCancel = TD->hasCancel();
1332 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1333 HasCancel = TD->hasCancel();
1334 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1335 HasCancel = TD->hasCancel();
1336 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1337 HasCancel = TD->hasCancel();
1338
1339 CodeGenFunction CGF(CGM, true);
1340 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1341 InnermostKind, HasCancel, Action);
1342 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1343 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1344 if (!Tied)
1345 NumberOfParts = Action.getNumberOfParts();
1346 return Res;
1347}
1348
1349static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1350 const RecordDecl *RD, const CGRecordLayout &RL,
1351 ArrayRef<llvm::Constant *> Data) {
1352 llvm::StructType *StructTy = RL.getLLVMType();
1353 unsigned PrevIdx = 0;
1354 ConstantInitBuilder CIBuilder(CGM);
1355 auto DI = Data.begin();
1356 for (const FieldDecl *FD : RD->fields()) {
1357 unsigned Idx = RL.getLLVMFieldNo(FD);
1358 // Fill the alignment.
1359 for (unsigned I = PrevIdx; I < Idx; ++I)
1360 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1361 PrevIdx = Idx + 1;
1362 Fields.add(*DI);
1363 ++DI;
1364 }
1365}
1366
1367template <class... As>
1368static llvm::GlobalVariable *
1369createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1370 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1371 As &&... Args) {
1372 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1373 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1374 ConstantInitBuilder CIBuilder(CGM);
1375 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1376 buildStructValue(Fields, CGM, RD, RL, Data);
1377 return Fields.finishAndCreateGlobal(
1378 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1379 std::forward<As>(Args)...);
1380}
1381
1382template <typename T>
1383static void
1384createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1385 ArrayRef<llvm::Constant *> Data,
1386 T &Parent) {
1387 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1388 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1389 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1390 buildStructValue(Fields, CGM, RD, RL, Data);
1391 Fields.finishAndAddTo(Parent);
1392}
1393
1394void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1395 bool AtCurrentPoint) {
1396 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1397 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1397, __PRETTY_FUNCTION__))
;
1398
1399 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1400 if (AtCurrentPoint) {
1401 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1402 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1403 } else {
1404 Elem.second.ServiceInsertPt =
1405 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1406 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1407 }
1408}
1409
1410void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1411 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1412 if (Elem.second.ServiceInsertPt) {
1413 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1414 Elem.second.ServiceInsertPt = nullptr;
1415 Ptr->eraseFromParent();
1416 }
1417}
1418
1419static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1420 SourceLocation Loc,
1421 SmallString<128> &Buffer) {
1422 llvm::raw_svector_ostream OS(Buffer);
1423 // Build debug location
1424 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1425 OS << ";" << PLoc.getFilename() << ";";
1426 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1427 OS << FD->getQualifiedNameAsString();
1428 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1429 return OS.str();
1430}
1431
1432llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1433 SourceLocation Loc,
1434 unsigned Flags) {
1435 llvm::Constant *SrcLocStr;
1436 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1437 Loc.isInvalid()) {
1438 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr();
1439 } else {
1440 std::string FunctionName = "";
1441 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1442 FunctionName = FD->getQualifiedNameAsString();
1443 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1444 const char *FileName = PLoc.getFilename();
1445 unsigned Line = PLoc.getLine();
1446 unsigned Column = PLoc.getColumn();
1447 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName.c_str(), FileName,
1448 Line, Column);
1449 }
1450 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1451 return OMPBuilder.getOrCreateIdent(SrcLocStr, llvm::omp::IdentFlag(Flags),
1452 Reserved2Flags);
1453}
1454
1455llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1456 SourceLocation Loc) {
1457 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1457, __PRETTY_FUNCTION__))
;
1458 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1459 // the clang invariants used below might be broken.
1460 if (CGM.getLangOpts().OpenMPIRBuilder) {
1461 SmallString<128> Buffer;
1462 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1463 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1464 getIdentStringFromSourceLocation(CGF, Loc, Buffer));
1465 return OMPBuilder.getOrCreateThreadID(
1466 OMPBuilder.getOrCreateIdent(SrcLocStr));
1467 }
1468
1469 llvm::Value *ThreadID = nullptr;
1470 // Check whether we've already cached a load of the thread id in this
1471 // function.
1472 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1473 if (I != OpenMPLocThreadIDMap.end()) {
1474 ThreadID = I->second.ThreadID;
1475 if (ThreadID != nullptr)
1476 return ThreadID;
1477 }
1478 // If exceptions are enabled, do not use parameter to avoid possible crash.
1479 if (auto *OMPRegionInfo =
1480 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1481 if (OMPRegionInfo->getThreadIDVariable()) {
1482 // Check if this an outlined function with thread id passed as argument.
1483 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1484 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1485 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1486 !CGF.getLangOpts().CXXExceptions ||
1487 CGF.Builder.GetInsertBlock() == TopBlock ||
1488 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1489 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1490 TopBlock ||
1491 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1492 CGF.Builder.GetInsertBlock()) {
1493 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1494 // If value loaded in entry block, cache it and use it everywhere in
1495 // function.
1496 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1497 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1498 Elem.second.ThreadID = ThreadID;
1499 }
1500 return ThreadID;
1501 }
1502 }
1503 }
1504
1505 // This is not an outlined function region - need to call __kmpc_int32
1506 // kmpc_global_thread_num(ident_t *loc).
1507 // Generate thread id value and cache this value for use across the
1508 // function.
1509 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1510 if (!Elem.second.ServiceInsertPt)
1511 setLocThreadIdInsertPt(CGF);
1512 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1513 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1514 llvm::CallInst *Call = CGF.Builder.CreateCall(
1515 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1516 OMPRTL___kmpc_global_thread_num),
1517 emitUpdateLocation(CGF, Loc));
1518 Call->setCallingConv(CGF.getRuntimeCC());
1519 Elem.second.ThreadID = Call;
1520 return Call;
1521}
1522
1523void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1524 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1524, __PRETTY_FUNCTION__))
;
1525 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1526 clearLocThreadIdInsertPt(CGF);
1527 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1528 }
1529 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1530 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1531 UDRMap.erase(D);
1532 FunctionUDRMap.erase(CGF.CurFn);
1533 }
1534 auto I = FunctionUDMMap.find(CGF.CurFn);
1535 if (I != FunctionUDMMap.end()) {
1536 for(const auto *D : I->second)
1537 UDMMap.erase(D);
1538 FunctionUDMMap.erase(I);
1539 }
1540 LastprivateConditionalToTypes.erase(CGF.CurFn);
1541 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1542}
1543
1544llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1545 return OMPBuilder.IdentPtr;
1546}
1547
1548llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1549 if (!Kmpc_MicroTy) {
1550 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1551 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1552 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1553 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1554 }
1555 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1556}
1557
1558llvm::FunctionCallee
1559CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
1560 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1561, __PRETTY_FUNCTION__))
1561 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1561, __PRETTY_FUNCTION__))
;
1562 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
1563 : "__kmpc_for_static_init_4u")
1564 : (IVSigned ? "__kmpc_for_static_init_8"
1565 : "__kmpc_for_static_init_8u");
1566 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1567 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1568 llvm::Type *TypeParams[] = {
1569 getIdentTyPointerTy(), // loc
1570 CGM.Int32Ty, // tid
1571 CGM.Int32Ty, // schedtype
1572 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1573 PtrTy, // p_lower
1574 PtrTy, // p_upper
1575 PtrTy, // p_stride
1576 ITy, // incr
1577 ITy // chunk
1578 };
1579 auto *FnTy =
1580 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1581 return CGM.CreateRuntimeFunction(FnTy, Name);
1582}
1583
1584llvm::FunctionCallee
1585CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
1586 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1587, __PRETTY_FUNCTION__))
1587 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588 StringRef Name =
1589 IVSize == 32
1590 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
1591 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
1592 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1593 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
1594 CGM.Int32Ty, // tid
1595 CGM.Int32Ty, // schedtype
1596 ITy, // lower
1597 ITy, // upper
1598 ITy, // stride
1599 ITy // chunk
1600 };
1601 auto *FnTy =
1602 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1603 return CGM.CreateRuntimeFunction(FnTy, Name);
1604}
1605
1606llvm::FunctionCallee
1607CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
1608 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1609, __PRETTY_FUNCTION__))
1609 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1609, __PRETTY_FUNCTION__))
;
1610 StringRef Name =
1611 IVSize == 32
1612 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
1613 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
1614 llvm::Type *TypeParams[] = {
1615 getIdentTyPointerTy(), // loc
1616 CGM.Int32Ty, // tid
1617 };
1618 auto *FnTy =
1619 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1620 return CGM.CreateRuntimeFunction(FnTy, Name);
1621}
1622
1623llvm::FunctionCallee
1624CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
1625 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1626, __PRETTY_FUNCTION__))
1626 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1626, __PRETTY_FUNCTION__))
;
1627 StringRef Name =
1628 IVSize == 32
1629 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
1630 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
1631 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1632 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1633 llvm::Type *TypeParams[] = {
1634 getIdentTyPointerTy(), // loc
1635 CGM.Int32Ty, // tid
1636 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1637 PtrTy, // p_lower
1638 PtrTy, // p_upper
1639 PtrTy // p_stride
1640 };
1641 auto *FnTy =
1642 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1643 return CGM.CreateRuntimeFunction(FnTy, Name);
1644}
1645
1646/// Obtain information that uniquely identifies a target entry. This
1647/// consists of the file and device IDs as well as line number associated with
1648/// the relevant entry source location.
1649static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
1650 unsigned &DeviceID, unsigned &FileID,
1651 unsigned &LineNum) {
1652 SourceManager &SM = C.getSourceManager();
1653
1654 // The loc should be always valid and have a file ID (the user cannot use
1655 // #pragma directives in macros)
1656
1657 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1657, __PRETTY_FUNCTION__))
;
1658
1659 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
1660 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1660, __PRETTY_FUNCTION__))
;
1661
1662 llvm::sys::fs::UniqueID ID;
1663 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) {
1664 PLoc = SM.getPresumedLoc(Loc, /*UseLineDirectives=*/false);
1665 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1665, __PRETTY_FUNCTION__))
;
1666 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
1667 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1668 << PLoc.getFilename() << EC.message();
1669 }
1670
1671 DeviceID = ID.getDevice();
1672 FileID = ID.getFile();
1673 LineNum = PLoc.getLine();
1674}
1675
1676Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1677 if (CGM.getLangOpts().OpenMPSimd)
1678 return Address::invalid();
1679 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1680 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1681 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1682 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1683 HasRequiresUnifiedSharedMemory))) {
1684 SmallString<64> PtrName;
1685 {
1686 llvm::raw_svector_ostream OS(PtrName);
1687 OS << CGM.getMangledName(GlobalDecl(VD));
1688 if (!VD->isExternallyVisible()) {
1689 unsigned DeviceID, FileID, Line;
1690 getTargetEntryUniqueInfo(CGM.getContext(),
1691 VD->getCanonicalDecl()->getBeginLoc(),
1692 DeviceID, FileID, Line);
1693 OS << llvm::format("_%x", FileID);
1694 }
1695 OS << "_decl_tgt_ref_ptr";
1696 }
1697 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
1698 if (!Ptr) {
1699 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
1700 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
1701 PtrName);
1702
1703 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1704 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1705
1706 if (!CGM.getLangOpts().OpenMPIsDevice)
1707 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
1708 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
1709 }
1710 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
1711 }
1712 return Address::invalid();
1713}
1714
1715llvm::Constant *
1716CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1717 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1718, __PRETTY_FUNCTION__))
1718 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1718, __PRETTY_FUNCTION__))
;
1719 // Lookup the entry, lazily creating it if necessary.
1720 std::string Suffix = getName({"cache", ""});
1721 return getOrCreateInternalVariable(
1722 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
1723}
1724
1725Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1726 const VarDecl *VD,
1727 Address VDAddr,
1728 SourceLocation Loc) {
1729 if (CGM.getLangOpts().OpenMPUseTLS &&
1730 CGM.getContext().getTargetInfo().isTLSSupported())
1731 return VDAddr;
1732
1733 llvm::Type *VarTy = VDAddr.getElementType();
1734 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1735 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1736 CGM.Int8PtrTy),
1737 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1738 getOrCreateThreadPrivateCache(VD)};
1739 return Address(CGF.EmitRuntimeCall(
1740 OMPBuilder.getOrCreateRuntimeFunction(
1741 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1742 Args),
1743 VDAddr.getAlignment());
1744}
1745
1746void CGOpenMPRuntime::emitThreadPrivateVarInit(
1747 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1748 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1749 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1750 // library.
1751 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1752 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1753 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1754 OMPLoc);
1755 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1756 // to register constructor/destructor for variable.
1757 llvm::Value *Args[] = {
1758 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
1759 Ctor, CopyCtor, Dtor};
1760 CGF.EmitRuntimeCall(
1761 OMPBuilder.getOrCreateRuntimeFunction(
1762 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1763 Args);
1764}
1765
1766llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1767 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1768 bool PerformInit, CodeGenFunction *CGF) {
1769 if (CGM.getLangOpts().OpenMPUseTLS &&
1770 CGM.getContext().getTargetInfo().isTLSSupported())
1771 return nullptr;
1772
1773 VD = VD->getDefinition(CGM.getContext());
1774 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
1775 QualType ASTTy = VD->getType();
1776
1777 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1778 const Expr *Init = VD->getAnyInitializer();
1779 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1780 // Generate function that re-emits the declaration's initializer into the
1781 // threadprivate copy of the variable VD
1782 CodeGenFunction CtorCGF(CGM);
1783 FunctionArgList Args;
1784 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1785 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1786 ImplicitParamDecl::Other);
1787 Args.push_back(&Dst);
1788
1789 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1790 CGM.getContext().VoidPtrTy, Args);
1791 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1792 std::string Name = getName({"__kmpc_global_ctor_", ""});
1793 llvm::Function *Fn =
1794 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1795 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1796 Args, Loc, Loc);
1797 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1798 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1799 CGM.getContext().VoidPtrTy, Dst.getLocation());
1800 Address Arg = Address(ArgVal, VDAddr.getAlignment());
1801 Arg = CtorCGF.Builder.CreateElementBitCast(
1802 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
1803 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1804 /*IsInitializer=*/true);
1805 ArgVal = CtorCGF.EmitLoadOfScalar(
1806 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1807 CGM.getContext().VoidPtrTy, Dst.getLocation());
1808 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1809 CtorCGF.FinishFunction();
1810 Ctor = Fn;
1811 }
1812 if (VD->getType().isDestructedType() != QualType::DK_none) {
1813 // Generate function that emits destructor call for the threadprivate copy
1814 // of the variable VD
1815 CodeGenFunction DtorCGF(CGM);
1816 FunctionArgList Args;
1817 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1818 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1819 ImplicitParamDecl::Other);
1820 Args.push_back(&Dst);
1821
1822 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1823 CGM.getContext().VoidTy, Args);
1824 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1825 std::string Name = getName({"__kmpc_global_dtor_", ""});
1826 llvm::Function *Fn =
1827 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1828 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1829 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1830 Loc, Loc);
1831 // Create a scope with an artificial location for the body of this function.
1832 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1833 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1834 DtorCGF.GetAddrOfLocalVar(&Dst),
1835 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
1836 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
1837 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1838 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1839 DtorCGF.FinishFunction();
1840 Dtor = Fn;
1841 }
1842 // Do not emit init function if it is not required.
1843 if (!Ctor && !Dtor)
1844 return nullptr;
1845
1846 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1847 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1848 /*isVarArg=*/false)
1849 ->getPointerTo();
1850 // Copying constructor for the threadprivate variable.
1851 // Must be NULL - reserved by runtime, but currently it requires that this
1852 // parameter is always NULL. Otherwise it fires assertion.
1853 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1854 if (Ctor == nullptr) {
1855 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1856 /*isVarArg=*/false)
1857 ->getPointerTo();
1858 Ctor = llvm::Constant::getNullValue(CtorTy);
1859 }
1860 if (Dtor == nullptr) {
1861 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1862 /*isVarArg=*/false)
1863 ->getPointerTo();
1864 Dtor = llvm::Constant::getNullValue(DtorTy);
1865 }
1866 if (!CGF) {
1867 auto *InitFunctionTy =
1868 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1869 std::string Name = getName({"__omp_threadprivate_init_", ""});
1870 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1871 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
1872 CodeGenFunction InitCGF(CGM);
1873 FunctionArgList ArgList;
1874 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1875 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1876 Loc, Loc);
1877 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1878 InitCGF.FinishFunction();
1879 return InitFunction;
1880 }
1881 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1882 }
1883 return nullptr;
1884}
1885
1886bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
1887 llvm::GlobalVariable *Addr,
1888 bool PerformInit) {
1889 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
1890 !CGM.getLangOpts().OpenMPIsDevice)
1891 return false;
1892 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1893 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1894 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1895 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1896 HasRequiresUnifiedSharedMemory))
1897 return CGM.getLangOpts().OpenMPIsDevice;
1898 VD = VD->getDefinition(CGM.getContext());
1899 assert(VD && "Unknown VarDecl")((VD && "Unknown VarDecl") ? static_cast<void> (
0) : __assert_fail ("VD && \"Unknown VarDecl\"", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1899, __PRETTY_FUNCTION__))
;
1900
1901 if (!DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
1902 return CGM.getLangOpts().OpenMPIsDevice;
1903
1904 QualType ASTTy = VD->getType();
1905 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
1906
1907 // Produce the unique prefix to identify the new target regions. We use
1908 // the source location of the variable declaration which we know to not
1909 // conflict with any target region.
1910 unsigned DeviceID;
1911 unsigned FileID;
1912 unsigned Line;
1913 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
1914 SmallString<128> Buffer, Out;
1915 {
1916 llvm::raw_svector_ostream OS(Buffer);
1917 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
1918 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
1919 }
1920
1921 const Expr *Init = VD->getAnyInitializer();
1922 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1923 llvm::Constant *Ctor;
1924 llvm::Constant *ID;
1925 if (CGM.getLangOpts().OpenMPIsDevice) {
1926 // Generate function that re-emits the declaration's initializer into
1927 // the threadprivate copy of the variable VD
1928 CodeGenFunction CtorCGF(CGM);
1929
1930 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1931 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1932 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1933 FTy, Twine(Buffer, "_ctor"), FI, Loc);
1934 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
1935 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1936 FunctionArgList(), Loc, Loc);
1937 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
1938 CtorCGF.EmitAnyExprToMem(Init,
1939 Address(Addr, CGM.getContext().getDeclAlign(VD)),
1940 Init->getType().getQualifiers(),
1941 /*IsInitializer=*/true);
1942 CtorCGF.FinishFunction();
1943 Ctor = Fn;
1944 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1945 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
1946 } else {
1947 Ctor = new llvm::GlobalVariable(
1948 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1949 llvm::GlobalValue::PrivateLinkage,
1950 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
1951 ID = Ctor;
1952 }
1953
1954 // Register the information for the entry associated with the constructor.
1955 Out.clear();
1956 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1957 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
1958 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
1959 }
1960 if (VD->getType().isDestructedType() != QualType::DK_none) {
1961 llvm::Constant *Dtor;
1962 llvm::Constant *ID;
1963 if (CGM.getLangOpts().OpenMPIsDevice) {
1964 // Generate function that emits destructor call for the threadprivate
1965 // copy of the variable VD
1966 CodeGenFunction DtorCGF(CGM);
1967
1968 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1969 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1970 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1971 FTy, Twine(Buffer, "_dtor"), FI, Loc);
1972 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1973 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1974 FunctionArgList(), Loc, Loc);
1975 // Create a scope with an artificial location for the body of this
1976 // function.
1977 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1978 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
1979 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1980 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1981 DtorCGF.FinishFunction();
1982 Dtor = Fn;
1983 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1984 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
1985 } else {
1986 Dtor = new llvm::GlobalVariable(
1987 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1988 llvm::GlobalValue::PrivateLinkage,
1989 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
1990 ID = Dtor;
1991 }
1992 // Register the information for the entry associated with the destructor.
1993 Out.clear();
1994 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1995 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
1996 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
1997 }
1998 return CGM.getLangOpts().OpenMPIsDevice;
1999}
2000
2001Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2002 QualType VarType,
2003 StringRef Name) {
2004 std::string Suffix = getName({"artificial", ""});
2005 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2006 llvm::Value *GAddr =
2007 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2008 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
2009 CGM.getTarget().isTLSSupported()) {
2010 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
2011 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
2012 }
2013 std::string CacheSuffix = getName({"cache", ""});
2014 llvm::Value *Args[] = {
2015 emitUpdateLocation(CGF, SourceLocation()),
2016 getThreadID(CGF, SourceLocation()),
2017 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2018 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2019 /*isSigned=*/false),
2020 getOrCreateInternalVariable(
2021 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2022 return Address(
2023 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2024 CGF.EmitRuntimeCall(
2025 OMPBuilder.getOrCreateRuntimeFunction(
2026 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
2027 Args),
2028 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2029 CGM.getContext().getTypeAlignInChars(VarType));
2030}
2031
2032void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
2033 const RegionCodeGenTy &ThenGen,
2034 const RegionCodeGenTy &ElseGen) {
2035 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2036
2037 // If the condition constant folds and can be elided, try to avoid emitting
2038 // the condition and the dead arm of the if/else.
2039 bool CondConstant;
2040 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2041 if (CondConstant)
2042 ThenGen(CGF);
2043 else
2044 ElseGen(CGF);
2045 return;
2046 }
2047
2048 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2049 // emit the conditional branch.
2050 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2051 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2052 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2053 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2054
2055 // Emit the 'then' code.
2056 CGF.EmitBlock(ThenBlock);
2057 ThenGen(CGF);
2058 CGF.EmitBranch(ContBlock);
2059 // Emit the 'else' code if present.
2060 // There is no need to emit line number for unconditional branch.
2061 (void)ApplyDebugLocation::CreateEmpty(CGF);
2062 CGF.EmitBlock(ElseBlock);
2063 ElseGen(CGF);
2064 // There is no need to emit line number for unconditional branch.
2065 (void)ApplyDebugLocation::CreateEmpty(CGF);
2066 CGF.EmitBranch(ContBlock);
2067 // Emit the continuation block for code after the if.
2068 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2069}
2070
2071void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2072 llvm::Function *OutlinedFn,
2073 ArrayRef<llvm::Value *> CapturedVars,
2074 const Expr *IfCond) {
2075 if (!CGF.HaveInsertPoint())
2076 return;
2077 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2078 auto &M = CGM.getModule();
2079 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2080 this](CodeGenFunction &CGF, PrePostActionTy &) {
2081 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2082 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2083 llvm::Value *Args[] = {
2084 RTLoc,
2085 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2086 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2087 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2088 RealArgs.append(std::begin(Args), std::end(Args));
2089 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2090
2091 llvm::FunctionCallee RTLFn =
2092 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2093 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2094 };
2095 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2096 this](CodeGenFunction &CGF, PrePostActionTy &) {
2097 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2098 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2099 // Build calls:
2100 // __kmpc_serialized_parallel(&Loc, GTid);
2101 llvm::Value *Args[] = {RTLoc, ThreadID};
2102 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2103 M, OMPRTL___kmpc_serialized_parallel),
2104 Args);
2105
2106 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2107 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2108 Address ZeroAddrBound =
2109 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2110 /*Name=*/".bound.zero.addr");
2111 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
2112 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2113 // ThreadId for serialized parallels is 0.
2114 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2115 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
2116 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2117
2118 // Ensure we do not inline the function. This is trivially true for the ones
2119 // passed to __kmpc_fork_call but the ones calles in serialized regions
2120 // could be inlined. This is not a perfect but it is closer to the invariant
2121 // we want, namely, every data environment starts with a new function.
2122 // TODO: We should pass the if condition to the runtime function and do the
2123 // handling there. Much cleaner code.
2124 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2125 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2126
2127 // __kmpc_end_serialized_parallel(&Loc, GTid);
2128 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2129 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2130 M, OMPRTL___kmpc_end_serialized_parallel),
2131 EndArgs);
2132 };
2133 if (IfCond) {
2134 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2135 } else {
2136 RegionCodeGenTy ThenRCG(ThenGen);
2137 ThenRCG(CGF);
2138 }
2139}
2140
2141// If we're inside an (outlined) parallel region, use the region info's
2142// thread-ID variable (it is passed in a first argument of the outlined function
2143// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2144// regular serial code region, get thread ID by calling kmp_int32
2145// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2146// return the address of that temp.
2147Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2148 SourceLocation Loc) {
2149 if (auto *OMPRegionInfo =
2150 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2151 if (OMPRegionInfo->getThreadIDVariable())
2152 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2153
2154 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2155 QualType Int32Ty =
2156 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2157 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2158 CGF.EmitStoreOfScalar(ThreadID,
2159 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2160
2161 return ThreadIDTemp;
2162}
2163
2164llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
2165 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2166 SmallString<256> Buffer;
2167 llvm::raw_svector_ostream Out(Buffer);
2168 Out << Name;
2169 StringRef RuntimeName = Out.str();
2170 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2171 if (Elem.second) {
2172 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2173, __PRETTY_FUNCTION__))
2173 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2173, __PRETTY_FUNCTION__))
;
2174 return &*Elem.second;
2175 }
2176
2177 return Elem.second = new llvm::GlobalVariable(
2178 CGM.getModule(), Ty, /*IsConstant*/ false,
2179 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2180 Elem.first(), /*InsertBefore=*/nullptr,
2181 llvm::GlobalValue::NotThreadLocal, AddressSpace);
2182}
2183
2184llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2185 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2186 std::string Name = getName({Prefix, "var"});
2187 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2188}
2189
2190namespace {
2191/// Common pre(post)-action for different OpenMP constructs.
2192class CommonActionTy final : public PrePostActionTy {
2193 llvm::FunctionCallee EnterCallee;
2194 ArrayRef<llvm::Value *> EnterArgs;
2195 llvm::FunctionCallee ExitCallee;
2196 ArrayRef<llvm::Value *> ExitArgs;
2197 bool Conditional;
2198 llvm::BasicBlock *ContBlock = nullptr;
2199
2200public:
2201 CommonActionTy(llvm::FunctionCallee EnterCallee,
2202 ArrayRef<llvm::Value *> EnterArgs,
2203 llvm::FunctionCallee ExitCallee,
2204 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2205 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2206 ExitArgs(ExitArgs), Conditional(Conditional) {}
2207 void Enter(CodeGenFunction &CGF) override {
2208 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2209 if (Conditional) {
2210 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2211 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2212 ContBlock = CGF.createBasicBlock("omp_if.end");
2213 // Generate the branch (If-stmt)
2214 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2215 CGF.EmitBlock(ThenBlock);
2216 }
2217 }
2218 void Done(CodeGenFunction &CGF) {
2219 // Emit the rest of blocks/branches
2220 CGF.EmitBranch(ContBlock);
2221 CGF.EmitBlock(ContBlock, true);
2222 }
2223 void Exit(CodeGenFunction &CGF) override {
2224 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2225 }
2226};
2227} // anonymous namespace
2228
2229void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2230 StringRef CriticalName,
2231 const RegionCodeGenTy &CriticalOpGen,
2232 SourceLocation Loc, const Expr *Hint) {
2233 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2234 // CriticalOpGen();
2235 // __kmpc_end_critical(ident_t *, gtid, Lock);
2236 // Prepare arguments and build a call to __kmpc_critical
2237 if (!CGF.HaveInsertPoint())
2238 return;
2239 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2240 getCriticalRegionLock(CriticalName)};
2241 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2242 std::end(Args));
2243 if (Hint) {
2244 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2245 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2246 }
2247 CommonActionTy Action(
2248 OMPBuilder.getOrCreateRuntimeFunction(
2249 CGM.getModule(),
2250 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2251 EnterArgs,
2252 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2253 OMPRTL___kmpc_end_critical),
2254 Args);
2255 CriticalOpGen.setAction(Action);
2256 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2257}
2258
2259void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2260 const RegionCodeGenTy &MasterOpGen,
2261 SourceLocation Loc) {
2262 if (!CGF.HaveInsertPoint())
2263 return;
2264 // if(__kmpc_master(ident_t *, gtid)) {
2265 // MasterOpGen();
2266 // __kmpc_end_master(ident_t *, gtid);
2267 // }
2268 // Prepare arguments and build a call to __kmpc_master
2269 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2270 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2271 CGM.getModule(), OMPRTL___kmpc_master),
2272 Args,
2273 OMPBuilder.getOrCreateRuntimeFunction(
2274 CGM.getModule(), OMPRTL___kmpc_end_master),
2275 Args,
2276 /*Conditional=*/true);
2277 MasterOpGen.setAction(Action);
2278 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2279 Action.Done(CGF);
2280}
2281
2282void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2283 SourceLocation Loc) {
2284 if (!CGF.HaveInsertPoint())
2285 return;
2286 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2287 OMPBuilder.createTaskyield(CGF.Builder);
2288 } else {
2289 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2290 llvm::Value *Args[] = {
2291 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2292 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
2293 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2294 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2295 Args);
2296 }
2297
2298 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2299 Region->emitUntiedSwitch(CGF);
2300}
2301
2302void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2303 const RegionCodeGenTy &TaskgroupOpGen,
2304 SourceLocation Loc) {
2305 if (!CGF.HaveInsertPoint())
2306 return;
2307 // __kmpc_taskgroup(ident_t *, gtid);
2308 // TaskgroupOpGen();
2309 // __kmpc_end_taskgroup(ident_t *, gtid);
2310 // Prepare arguments and build a call to __kmpc_taskgroup
2311 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2312 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2313 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2314 Args,
2315 OMPBuilder.getOrCreateRuntimeFunction(
2316 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2317 Args);
2318 TaskgroupOpGen.setAction(Action);
2319 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2320}
2321
2322/// Given an array of pointers to variables, project the address of a
2323/// given variable.
2324static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2325 unsigned Index, const VarDecl *Var) {
2326 // Pull out the pointer to the variable.
2327 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2328 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2329
2330 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
2331 Addr = CGF.Builder.CreateElementBitCast(
2332 Addr, CGF.ConvertTypeForMem(Var->getType()));
2333 return Addr;
2334}
2335
2336static llvm::Value *emitCopyprivateCopyFunction(
2337 CodeGenModule &CGM, llvm::Type *ArgsType,
2338 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2339 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2340 SourceLocation Loc) {
2341 ASTContext &C = CGM.getContext();
2342 // void copy_func(void *LHSArg, void *RHSArg);
2343 FunctionArgList Args;
2344 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2345 ImplicitParamDecl::Other);
2346 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2347 ImplicitParamDecl::Other);
2348 Args.push_back(&LHSArg);
2349 Args.push_back(&RHSArg);
2350 const auto &CGFI =
2351 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
2352 std::string Name =
2353 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
2354 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
2355 llvm::GlobalValue::InternalLinkage, Name,
2356 &CGM.getModule());
2357 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
2358 Fn->setDoesNotRecurse();
2359 CodeGenFunction CGF(CGM);
2360 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2361 // Dest = (void*[n])(LHSArg);
2362 // Src = (void*[n])(RHSArg);
2363 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2364 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2365 ArgsType), CGF.getPointerAlign());
2366 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2367 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2368 ArgsType), CGF.getPointerAlign());
2369 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2370 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2371 // ...
2372 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2373 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2374 const auto *DestVar =
2375 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2376 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2377
2378 const auto *SrcVar =
2379 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2380 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2381
2382 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2383 QualType Type = VD->getType();
2384 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2385 }
2386 CGF.FinishFunction();
2387 return Fn;
2388}
2389
2390void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2391 const RegionCodeGenTy &SingleOpGen,
2392 SourceLocation Loc,
2393 ArrayRef<const Expr *> CopyprivateVars,
2394 ArrayRef<const Expr *> SrcExprs,
2395 ArrayRef<const Expr *> DstExprs,
2396 ArrayRef<const Expr *> AssignmentOps) {
2397 if (!CGF.HaveInsertPoint())
2398 return;
2399 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2401, __PRETTY_FUNCTION__))
2400 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2401, __PRETTY_FUNCTION__))
2401 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2401, __PRETTY_FUNCTION__))
;
2402 ASTContext &C = CGM.getContext();
2403 // int32 did_it = 0;
2404 // if(__kmpc_single(ident_t *, gtid)) {
2405 // SingleOpGen();
2406 // __kmpc_end_single(ident_t *, gtid);
2407 // did_it = 1;
2408 // }
2409 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2410 // <copy_func>, did_it);
2411
2412 Address DidIt = Address::invalid();
2413 if (!CopyprivateVars.empty()) {
2414 // int32 did_it = 0;
2415 QualType KmpInt32Ty =
2416 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2417 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2418 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2419 }
2420 // Prepare arguments and build a call to __kmpc_single
2421 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2422 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2423 CGM.getModule(), OMPRTL___kmpc_single),
2424 Args,
2425 OMPBuilder.getOrCreateRuntimeFunction(
2426 CGM.getModule(), OMPRTL___kmpc_end_single),
2427 Args,
2428 /*Conditional=*/true);
2429 SingleOpGen.setAction(Action);
2430 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2431 if (DidIt.isValid()) {
2432 // did_it = 1;
2433 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2434 }
2435 Action.Done(CGF);
2436 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2437 // <copy_func>, did_it);
2438 if (DidIt.isValid()) {
2439 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2440 QualType CopyprivateArrayTy = C.getConstantArrayType(
2441 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
2442 /*IndexTypeQuals=*/0);
2443 // Create a list of all private variables for copyprivate.
2444 Address CopyprivateList =
2445 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2446 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2447 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2448 CGF.Builder.CreateStore(
2449 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2450 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2451 CGF.VoidPtrTy),
2452 Elem);
2453 }
2454 // Build function that copies private values from single region to all other
2455 // threads in the corresponding parallel region.
2456 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2457 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
2458 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
2459 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2460 Address CL =
2461 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2462 CGF.VoidPtrTy);
2463 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2464 llvm::Value *Args[] = {
2465 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2466 getThreadID(CGF, Loc), // i32 <gtid>
2467 BufSize, // size_t <buf_size>
2468 CL.getPointer(), // void *<copyprivate list>
2469 CpyFn, // void (*) (void *, void *) <copy_func>
2470 DidItVal // i32 did_it
2471 };
2472 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2473 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2474 Args);
2475 }
2476}
2477
2478void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2479 const RegionCodeGenTy &OrderedOpGen,
2480 SourceLocation Loc, bool IsThreads) {
2481 if (!CGF.HaveInsertPoint())
2482 return;
2483 // __kmpc_ordered(ident_t *, gtid);
2484 // OrderedOpGen();
2485 // __kmpc_end_ordered(ident_t *, gtid);
2486 // Prepare arguments and build a call to __kmpc_ordered
2487 if (IsThreads) {
2488 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2489 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2490 CGM.getModule(), OMPRTL___kmpc_ordered),
2491 Args,
2492 OMPBuilder.getOrCreateRuntimeFunction(
2493 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2494 Args);
2495 OrderedOpGen.setAction(Action);
2496 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2497 return;
2498 }
2499 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2500}
2501
2502unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2503 unsigned Flags;
2504 if (Kind == OMPD_for)
2505 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2506 else if (Kind == OMPD_sections)
2507 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2508 else if (Kind == OMPD_single)
2509 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2510 else if (Kind == OMPD_barrier)
2511 Flags = OMP_IDENT_BARRIER_EXPL;
2512 else
2513 Flags = OMP_IDENT_BARRIER_IMPL;
2514 return Flags;
2515}
2516
2517void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2518 CodeGenFunction &CGF, const OMPLoopDirective &S,
2519 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2520 // Check if the loop directive is actually a doacross loop directive. In this
2521 // case choose static, 1 schedule.
2522 if (llvm::any_of(
2523 S.getClausesOfKind<OMPOrderedClause>(),
2524 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2525 ScheduleKind = OMPC_SCHEDULE_static;
2526 // Chunk size is 1 in this case.
2527 llvm::APInt ChunkSize(32, 1);
2528 ChunkExpr = IntegerLiteral::Create(
2529 CGF.getContext(), ChunkSize,
2530 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2531 SourceLocation());
2532 }
2533}
2534
2535void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2536 OpenMPDirectiveKind Kind, bool EmitChecks,
2537 bool ForceSimpleCall) {
2538 // Check if we should use the OMPBuilder
2539 auto *OMPRegionInfo =
2540 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2541 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2542 CGF.Builder.restoreIP(OMPBuilder.createBarrier(
2543 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
2544 return;
2545 }
2546
2547 if (!CGF.HaveInsertPoint())
2548 return;
2549 // Build call __kmpc_cancel_barrier(loc, thread_id);
2550 // Build call __kmpc_barrier(loc, thread_id);
2551 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2552 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2553 // thread_id);
2554 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2555 getThreadID(CGF, Loc)};
2556 if (OMPRegionInfo) {
2557 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2558 llvm::Value *Result = CGF.EmitRuntimeCall(
2559 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2560 OMPRTL___kmpc_cancel_barrier),
2561 Args);
2562 if (EmitChecks) {
2563 // if (__kmpc_cancel_barrier()) {
2564 // exit from construct;
2565 // }
2566 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2567 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2568 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2569 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2570 CGF.EmitBlock(ExitBB);
2571 // exit from construct;
2572 CodeGenFunction::JumpDest CancelDestination =
2573 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2574 CGF.EmitBranchThroughCleanup(CancelDestination);
2575 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2576 }
2577 return;
2578 }
2579 }
2580 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2581 CGM.getModule(), OMPRTL___kmpc_barrier),
2582 Args);
2583}
2584
2585/// Map the OpenMP loop schedule to the runtime enumeration.
2586static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2587 bool Chunked, bool Ordered) {
2588 switch (ScheduleKind) {
2589 case OMPC_SCHEDULE_static:
2590 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2591 : (Ordered ? OMP_ord_static : OMP_sch_static);
2592 case OMPC_SCHEDULE_dynamic:
2593 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2594 case OMPC_SCHEDULE_guided:
2595 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2596 case OMPC_SCHEDULE_runtime:
2597 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2598 case OMPC_SCHEDULE_auto:
2599 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2600 case OMPC_SCHEDULE_unknown:
2601 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2601, __PRETTY_FUNCTION__))
;
2602 return Ordered ? OMP_ord_static : OMP_sch_static;
2603 }
2604 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2604)
;
2605}
2606
2607/// Map the OpenMP distribute schedule to the runtime enumeration.
2608static OpenMPSchedType
2609getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2610 // only static is allowed for dist_schedule
2611 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2612}
2613
2614bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2615 bool Chunked) const {
2616 OpenMPSchedType Schedule =
2617 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2618 return Schedule == OMP_sch_static;
2619}
2620
2621bool CGOpenMPRuntime::isStaticNonchunked(
2622 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2623 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2624 return Schedule == OMP_dist_sch_static;
2625}
2626
2627bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2628 bool Chunked) const {
2629 OpenMPSchedType Schedule =
2630 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2631 return Schedule == OMP_sch_static_chunked;
2632}
2633
2634bool CGOpenMPRuntime::isStaticChunked(
2635 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2636 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2637 return Schedule == OMP_dist_sch_static_chunked;
2638}
2639
2640bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2641 OpenMPSchedType Schedule =
2642 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2643 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2643, __PRETTY_FUNCTION__))
;
2644 return Schedule != OMP_sch_static;
2645}
2646
2647static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2648 OpenMPScheduleClauseModifier M1,
2649 OpenMPScheduleClauseModifier M2) {
2650 int Modifier = 0;
2651 switch (M1) {
2652 case OMPC_SCHEDULE_MODIFIER_monotonic:
2653 Modifier = OMP_sch_modifier_monotonic;
2654 break;
2655 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2656 Modifier = OMP_sch_modifier_nonmonotonic;
2657 break;
2658 case OMPC_SCHEDULE_MODIFIER_simd:
2659 if (Schedule == OMP_sch_static_chunked)
2660 Schedule = OMP_sch_static_balanced_chunked;
2661 break;
2662 case OMPC_SCHEDULE_MODIFIER_last:
2663 case OMPC_SCHEDULE_MODIFIER_unknown:
2664 break;
2665 }
2666 switch (M2) {
2667 case OMPC_SCHEDULE_MODIFIER_monotonic:
2668 Modifier = OMP_sch_modifier_monotonic;
2669 break;
2670 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2671 Modifier = OMP_sch_modifier_nonmonotonic;
2672 break;
2673 case OMPC_SCHEDULE_MODIFIER_simd:
2674 if (Schedule == OMP_sch_static_chunked)
2675 Schedule = OMP_sch_static_balanced_chunked;
2676 break;
2677 case OMPC_SCHEDULE_MODIFIER_last:
2678 case OMPC_SCHEDULE_MODIFIER_unknown:
2679 break;
2680 }
2681 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2682 // If the static schedule kind is specified or if the ordered clause is
2683 // specified, and if the nonmonotonic modifier is not specified, the effect is
2684 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2685 // modifier is specified, the effect is as if the nonmonotonic modifier is
2686 // specified.
2687 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2688 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2689 Schedule == OMP_sch_static_balanced_chunked ||
2690 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2691 Schedule == OMP_dist_sch_static_chunked ||
2692 Schedule == OMP_dist_sch_static))
2693 Modifier = OMP_sch_modifier_nonmonotonic;
2694 }
2695 return Schedule | Modifier;
2696}
2697
2698void CGOpenMPRuntime::emitForDispatchInit(
2699 CodeGenFunction &CGF, SourceLocation Loc,
2700 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2701 bool Ordered, const DispatchRTInput &DispatchValues) {
2702 if (!CGF.HaveInsertPoint())
2703 return;
2704 OpenMPSchedType Schedule = getRuntimeSchedule(
2705 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
2706 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2709, __PRETTY_FUNCTION__))
2707 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2709, __PRETTY_FUNCTION__))
2708 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2709, __PRETTY_FUNCTION__))
2709 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2709, __PRETTY_FUNCTION__))
;
2710 // Call __kmpc_dispatch_init(
2711 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2712 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2713 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2714
2715 // If the Chunk was not specified in the clause - use default value 1.
2716 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2717 : CGF.Builder.getIntN(IVSize, 1);
2718 llvm::Value *Args[] = {
2719 emitUpdateLocation(CGF, Loc),
2720 getThreadID(CGF, Loc),
2721 CGF.Builder.getInt32(addMonoNonMonoModifier(
2722 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
2723 DispatchValues.LB, // Lower
2724 DispatchValues.UB, // Upper
2725 CGF.Builder.getIntN(IVSize, 1), // Stride
2726 Chunk // Chunk
2727 };
2728 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2729}
2730
2731static void emitForStaticInitCall(
2732 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2733 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2734 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2735 const CGOpenMPRuntime::StaticRTInput &Values) {
2736 if (!CGF.HaveInsertPoint())
2737 return;
2738
2739 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
;
2740 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2744, __PRETTY_FUNCTION__))
2741 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2744, __PRETTY_FUNCTION__))
2742 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2744, __PRETTY_FUNCTION__))
2743 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2744, __PRETTY_FUNCTION__))
2744 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2744, __PRETTY_FUNCTION__))
;
2745
2746 // Call __kmpc_for_static_init(
2747 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2748 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2749 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2750 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2751 llvm::Value *Chunk = Values.Chunk;
2752 if (Chunk == nullptr) {
2753 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2755, __PRETTY_FUNCTION__))
2754 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2755, __PRETTY_FUNCTION__))
2755 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2755, __PRETTY_FUNCTION__))
;
2756 // If the Chunk was not specified in the clause - use default value 1.
2757 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2758 } else {
2759 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2763, __PRETTY_FUNCTION__))
2760 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2763, __PRETTY_FUNCTION__))
2761 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2763, __PRETTY_FUNCTION__))
2762 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2763, __PRETTY_FUNCTION__))
2763 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2763, __PRETTY_FUNCTION__))
;
2764 }
2765 llvm::Value *Args[] = {
2766 UpdateLocation,
2767 ThreadId,
2768 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2769 M2)), // Schedule type
2770 Values.IL.getPointer(), // &isLastIter
2771 Values.LB.getPointer(), // &LB
2772 Values.UB.getPointer(), // &UB
2773 Values.ST.getPointer(), // &Stride
2774 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2775 Chunk // Chunk
2776 };
2777 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2778}
2779
2780void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2781 SourceLocation Loc,
2782 OpenMPDirectiveKind DKind,
2783 const OpenMPScheduleTy &ScheduleKind,
2784 const StaticRTInput &Values) {
2785 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
2786 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
2787 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2788, __PRETTY_FUNCTION__))
2788 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2788, __PRETTY_FUNCTION__))
;
2789 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2790 isOpenMPLoopDirective(DKind)
2791 ? OMP_IDENT_WORK_LOOP
2792 : OMP_IDENT_WORK_SECTIONS);
2793 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2794 llvm::FunctionCallee StaticInitFunction =
2795 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2796 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2797 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2798 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
2799}
2800
2801void CGOpenMPRuntime::emitDistributeStaticInit(
2802 CodeGenFunction &CGF, SourceLocation Loc,
2803 OpenMPDistScheduleClauseKind SchedKind,
2804 const CGOpenMPRuntime::StaticRTInput &Values) {
2805 OpenMPSchedType ScheduleNum =
2806 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
2807 llvm::Value *UpdatedLocation =
2808 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2809 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2810 llvm::FunctionCallee StaticInitFunction =
2811 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2812 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2813 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
2814 OMPC_SCHEDULE_MODIFIER_unknown, Values);
2815}
2816
2817void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2818 SourceLocation Loc,
2819 OpenMPDirectiveKind DKind) {
2820 if (!CGF.HaveInsertPoint())
2821 return;
2822 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2823 llvm::Value *Args[] = {
2824 emitUpdateLocation(CGF, Loc,
2825 isOpenMPDistributeDirective(DKind)
2826 ? OMP_IDENT_WORK_DISTRIBUTE
2827 : isOpenMPLoopDirective(DKind)
2828 ? OMP_IDENT_WORK_LOOP
2829 : OMP_IDENT_WORK_SECTIONS),
2830 getThreadID(CGF, Loc)};
2831 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2832 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2833 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2834 Args);
2835}
2836
2837void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2838 SourceLocation Loc,
2839 unsigned IVSize,
2840 bool IVSigned) {
2841 if (!CGF.HaveInsertPoint())
2842 return;
2843 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2844 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2845 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2846}
2847
2848llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2849 SourceLocation Loc, unsigned IVSize,
2850 bool IVSigned, Address IL,
2851 Address LB, Address UB,
2852 Address ST) {
2853 // Call __kmpc_dispatch_next(
2854 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2855 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2856 // kmp_int[32|64] *p_stride);
2857 llvm::Value *Args[] = {
2858 emitUpdateLocation(CGF, Loc),
2859 getThreadID(CGF, Loc),
2860 IL.getPointer(), // &isLastIter
2861 LB.getPointer(), // &Lower
2862 UB.getPointer(), // &Upper
2863 ST.getPointer() // &Stride
2864 };
2865 llvm::Value *Call =
2866 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2867 return CGF.EmitScalarConversion(
2868 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2869 CGF.getContext().BoolTy, Loc);
2870}
2871
2872void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2873 llvm::Value *NumThreads,
2874 SourceLocation Loc) {
2875 if (!CGF.HaveInsertPoint())
2876 return;
2877 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2878 llvm::Value *Args[] = {
2879 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2880 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2881 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2882 CGM.getModule(), OMPRTL___kmpc_push_num_threads),
2883 Args);
2884}
2885
2886void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2887 ProcBindKind ProcBind,
2888 SourceLocation Loc) {
2889 if (!CGF.HaveInsertPoint())
2890 return;
2891 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2891, __PRETTY_FUNCTION__))
;
2892 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2893 llvm::Value *Args[] = {
2894 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2895 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
2896 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2897 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2898 Args);
2899}
2900
2901void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2902 SourceLocation Loc, llvm::AtomicOrdering AO) {
2903 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2904 OMPBuilder.createFlush(CGF.Builder);
2905 } else {
2906 if (!CGF.HaveInsertPoint())
2907 return;
2908 // Build call void __kmpc_flush(ident_t *loc)
2909 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2910 CGM.getModule(), OMPRTL___kmpc_flush),
2911 emitUpdateLocation(CGF, Loc));
2912 }
2913}
2914
2915namespace {
2916/// Indexes of fields for type kmp_task_t.
2917enum KmpTaskTFields {
2918 /// List of shared variables.
2919 KmpTaskTShareds,
2920 /// Task routine.
2921 KmpTaskTRoutine,
2922 /// Partition id for the untied tasks.
2923 KmpTaskTPartId,
2924 /// Function with call of destructors for private variables.
2925 Data1,
2926 /// Task priority.
2927 Data2,
2928 /// (Taskloops only) Lower bound.
2929 KmpTaskTLowerBound,
2930 /// (Taskloops only) Upper bound.
2931 KmpTaskTUpperBound,
2932 /// (Taskloops only) Stride.
2933 KmpTaskTStride,
2934 /// (Taskloops only) Is last iteration flag.
2935 KmpTaskTLastIter,
2936 /// (Taskloops only) Reduction data.
2937 KmpTaskTReductions,
2938};
2939} // anonymous namespace
2940
2941bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
2942 return OffloadEntriesTargetRegion.empty() &&
2943 OffloadEntriesDeviceGlobalVar.empty();
2944}
2945
2946/// Initialize target region entry.
2947void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2948 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2949 StringRef ParentName, unsigned LineNum,
2950 unsigned Order) {
2951 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2953, __PRETTY_FUNCTION__))
2952 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2953, __PRETTY_FUNCTION__))
2953 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2953, __PRETTY_FUNCTION__))
;
2954 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
2955 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
2956 OMPTargetRegionEntryTargetRegion);
2957 ++OffloadingEntriesNum;
2958}
2959
2960void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2961 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2962 StringRef ParentName, unsigned LineNum,
2963 llvm::Constant *Addr, llvm::Constant *ID,
2964 OMPTargetRegionEntryKind Flags) {
2965 // If we are emitting code for a target, the entry is already initialized,
2966 // only has to be registered.
2967 if (CGM.getLangOpts().OpenMPIsDevice) {
2968 // This could happen if the device compilation is invoked standalone.
2969 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum))
2970 initializeTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2971 OffloadingEntriesNum);
2972 auto &Entry =
2973 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
2974 Entry.setAddress(Addr);
2975 Entry.setID(ID);
2976 Entry.setFlags(Flags);
2977 } else {
2978 if (Flags ==
2979 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion &&
2980 hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2981 /*IgnoreAddressId*/ true))
2982 return;
2983 assert(!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2984, __PRETTY_FUNCTION__))
2984 "Target region entry already registered!")((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2984, __PRETTY_FUNCTION__))
;
2985 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
2986 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
2987 ++OffloadingEntriesNum;
2988 }
2989}
2990
2991bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2992 unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum,
2993 bool IgnoreAddressId) const {
2994 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2995 if (PerDevice == OffloadEntriesTargetRegion.end())
2996 return false;
2997 auto PerFile = PerDevice->second.find(FileID);
2998 if (PerFile == PerDevice->second.end())
2999 return false;
3000 auto PerParentName = PerFile->second.find(ParentName);
3001 if (PerParentName == PerFile->second.end())
3002 return false;
3003 auto PerLine = PerParentName->second.find(LineNum);
3004 if (PerLine == PerParentName->second.end())
3005 return false;
3006 // Fail if this entry is already registered.
3007 if (!IgnoreAddressId &&
3008 (PerLine->second.getAddress() || PerLine->second.getID()))
3009 return false;
3010 return true;
3011}
3012
3013void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3014 const OffloadTargetRegionEntryInfoActTy &Action) {
3015 // Scan all target region entries and perform the provided action.
3016 for (const auto &D : OffloadEntriesTargetRegion)
3017 for (const auto &F : D.second)
3018 for (const auto &P : F.second)
3019 for (const auto &L : P.second)
3020 Action(D.first, F.first, P.first(), L.first, L.second);
3021}
3022
3023void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3024 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3025 OMPTargetGlobalVarEntryKind Flags,
3026 unsigned Order) {
3027 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3029, __PRETTY_FUNCTION__))
3028 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3029, __PRETTY_FUNCTION__))
3029 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3029, __PRETTY_FUNCTION__))
;
3030 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3031 ++OffloadingEntriesNum;
3032}
3033
3034void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3035 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3036 CharUnits VarSize,
3037 OMPTargetGlobalVarEntryKind Flags,
3038 llvm::GlobalValue::LinkageTypes Linkage) {
3039 if (CGM.getLangOpts().OpenMPIsDevice) {
3040 // This could happen if the device compilation is invoked standalone.
3041 if (!hasDeviceGlobalVarEntryInfo(VarName))
3042 initializeDeviceGlobalVarEntryInfo(VarName, Flags, OffloadingEntriesNum);
3043 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3044 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3045, __PRETTY_FUNCTION__))
3045 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3045, __PRETTY_FUNCTION__))
;
3046 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3047 if (Entry.getVarSize().isZero()) {
3048 Entry.setVarSize(VarSize);
3049 Entry.setLinkage(Linkage);
3050 }
3051 return;
3052 }
3053 Entry.setVarSize(VarSize);
3054 Entry.setLinkage(Linkage);
3055 Entry.setAddress(Addr);
3056 } else {
3057 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3058 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3059 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3060, __PRETTY_FUNCTION__))
3060 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3060, __PRETTY_FUNCTION__))
;
3061 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3062, __PRETTY_FUNCTION__))
3062 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3062, __PRETTY_FUNCTION__))
;
3063 if (Entry.getVarSize().isZero()) {
3064 Entry.setVarSize(VarSize);
3065 Entry.setLinkage(Linkage);
3066 }
3067 return;
3068 }
3069 OffloadEntriesDeviceGlobalVar.try_emplace(
3070 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3071 ++OffloadingEntriesNum;
3072 }
3073}
3074
3075void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3076 actOnDeviceGlobalVarEntriesInfo(
3077 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3078 // Scan all target region entries and perform the provided action.
3079 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3080 Action(E.getKey(), E.getValue());
3081}
3082
3083void CGOpenMPRuntime::createOffloadEntry(
3084 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3085 llvm::GlobalValue::LinkageTypes Linkage) {
3086 StringRef Name = Addr->getName();
3087 llvm::Module &M = CGM.getModule();
3088 llvm::LLVMContext &C = M.getContext();
3089
3090 // Create constant string with the name.
3091 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3092
3093 std::string StringName = getName({"omp_offloading", "entry_name"});
3094 auto *Str = new llvm::GlobalVariable(
3095 M, StrPtrInit->getType(), /*isConstant=*/true,
3096 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3097 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3098
3099 llvm::Constant *Data[] = {
3100 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(ID, CGM.VoidPtrTy),
3101 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, CGM.Int8PtrTy),
3102 llvm::ConstantInt::get(CGM.SizeTy, Size),
3103 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3104 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3105 std::string EntryName = getName({"omp_offloading", "entry", ""});
3106 llvm::GlobalVariable *Entry = createGlobalStruct(
3107 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
3108 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
3109
3110 // The entry has to be created in the section the linker expects it to be.
3111 Entry->setSection("omp_offloading_entries");
3112}
3113
3114void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3115 // Emit the offloading entries and metadata so that the device codegen side
3116 // can easily figure out what to emit. The produced metadata looks like
3117 // this:
3118 //
3119 // !omp_offload.info = !{!1, ...}
3120 //
3121 // Right now we only generate metadata for function that contain target
3122 // regions.
3123
3124 // If we are in simd mode or there are no entries, we don't need to do
3125 // anything.
3126 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
3127 return;
3128
3129 llvm::Module &M = CGM.getModule();
3130 llvm::LLVMContext &C = M.getContext();
3131 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
3132 SourceLocation, StringRef>,
3133 16>
3134 OrderedEntries(OffloadEntriesInfoManager.size());
3135 llvm::SmallVector<StringRef, 16> ParentFunctions(
3136 OffloadEntriesInfoManager.size());
3137
3138 // Auxiliary methods to create metadata values and strings.
3139 auto &&GetMDInt = [this](unsigned V) {
3140 return llvm::ConstantAsMetadata::get(
3141 llvm::ConstantInt::get(CGM.Int32Ty, V));
3142 };
3143
3144 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3145
3146 // Create the offloading info metadata node.
3147 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3148
3149 // Create function that emits metadata for each target region entry;
3150 auto &&TargetRegionMetadataEmitter =
3151 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
3152 &GetMDString](
3153 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3154 unsigned Line,
3155 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3156 // Generate metadata for target regions. Each entry of this metadata
3157 // contains:
3158 // - Entry 0 -> Kind of this type of metadata (0).
3159 // - Entry 1 -> Device ID of the file where the entry was identified.
3160 // - Entry 2 -> File ID of the file where the entry was identified.
3161 // - Entry 3 -> Mangled name of the function where the entry was
3162 // identified.
3163 // - Entry 4 -> Line in the file where the entry was identified.
3164 // - Entry 5 -> Order the entry was created.
3165 // The first element of the metadata node is the kind.
3166 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3167 GetMDInt(FileID), GetMDString(ParentName),
3168 GetMDInt(Line), GetMDInt(E.getOrder())};
3169
3170 SourceLocation Loc;
3171 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3172 E = CGM.getContext().getSourceManager().fileinfo_end();
3173 I != E; ++I) {
3174 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
3175 I->getFirst()->getUniqueID().getFile() == FileID) {
3176 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3177 I->getFirst(), Line, 1);
3178 break;
3179 }
3180 }
3181 // Save this entry in the right position of the ordered entries array.
3182 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
3183 ParentFunctions[E.getOrder()] = ParentName;
3184
3185 // Add metadata to the named metadata node.
3186 MD->addOperand(llvm::MDNode::get(C, Ops));
3187 };
3188
3189 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3190 TargetRegionMetadataEmitter);
3191
3192 // Create function that emits metadata for each device global variable entry;
3193 auto &&DeviceGlobalVarMetadataEmitter =
3194 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3195 MD](StringRef MangledName,
3196 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3197 &E) {
3198 // Generate metadata for global variables. Each entry of this metadata
3199 // contains:
3200 // - Entry 0 -> Kind of this type of metadata (1).
3201 // - Entry 1 -> Mangled name of the variable.
3202 // - Entry 2 -> Declare target kind.
3203 // - Entry 3 -> Order the entry was created.
3204 // The first element of the metadata node is the kind.
3205 llvm::Metadata *Ops[] = {
3206 GetMDInt(E.getKind()), GetMDString(MangledName),
3207 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3208
3209 // Save this entry in the right position of the ordered entries array.
3210 OrderedEntries[E.getOrder()] =
3211 std::make_tuple(&E, SourceLocation(), MangledName);
3212
3213 // Add metadata to the named metadata node.
3214 MD->addOperand(llvm::MDNode::get(C, Ops));
3215 };
3216
3217 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3218 DeviceGlobalVarMetadataEmitter);
3219
3220 for (const auto &E : OrderedEntries) {
3221 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3221, __PRETTY_FUNCTION__))
;
3222 if (const auto *CE =
3223 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3224 std::get<0>(E))) {
3225 if (!CE->getID() || !CE->getAddress()) {
3226 // Do not blame the entry if the parent funtion is not emitted.
3227 StringRef FnName = ParentFunctions[CE->getOrder()];
3228 if (!CGM.GetGlobalValue(FnName))
3229 continue;
3230 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3231 DiagnosticsEngine::Error,
3232 "Offloading entry for target region in %0 is incorrect: either the "
3233 "address or the ID is invalid.");
3234 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
3235 continue;
3236 }
3237 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
3238 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3239 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
3240 OffloadEntryInfoDeviceGlobalVar>(
3241 std::get<0>(E))) {
3242 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3243 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3244 CE->getFlags());
3245 switch (Flags) {
3246 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3247 if (CGM.getLangOpts().OpenMPIsDevice &&
3248 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
3249 continue;
3250 if (!CE->getAddress()) {
3251 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3252 DiagnosticsEngine::Error, "Offloading entry for declare target "
3253 "variable %0 is incorrect: the "
3254 "address is invalid.");
3255 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
3256 continue;
3257 }
3258 // The vaiable has no definition - no need to add the entry.
3259 if (CE->getVarSize().isZero())
3260 continue;
3261 break;
3262 }
3263 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3264 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3266, __PRETTY_FUNCTION__))
3265 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3266, __PRETTY_FUNCTION__))
3266 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3266, __PRETTY_FUNCTION__))
;
3267 if (CGM.getLangOpts().OpenMPIsDevice)
3268 continue;
3269 if (!CE->getAddress()) {
3270 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3271 DiagnosticsEngine::Error,
3272 "Offloading entry for declare target variable is incorrect: the "
3273 "address is invalid.");
3274 CGM.getDiags().Report(DiagID);
3275 continue;
3276 }
3277 break;
3278 }
3279 createOffloadEntry(CE->getAddress(), CE->getAddress(),
3280 CE->getVarSize().getQuantity(), Flags,
3281 CE->getLinkage());
3282 } else {
3283 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3283)
;
3284 }
3285 }
3286}
3287
3288/// Loads all the offload entries information from the host IR
3289/// metadata.
3290void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3291 // If we are in target mode, load the metadata from the host IR. This code has
3292 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
3293
3294 if (!CGM.getLangOpts().OpenMPIsDevice)
3295 return;
3296
3297 if (CGM.getLangOpts().OMPHostIRFile.empty())
3298 return;
3299
3300 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3301 if (auto EC = Buf.getError()) {
3302 CGM.getDiags().Report(diag::err_cannot_open_file)
3303 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3304 return;
3305 }
3306
3307 llvm::LLVMContext C;
3308 auto ME = expectedToErrorOrAndEmitErrors(
3309 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3310
3311 if (auto EC = ME.getError()) {
3312 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3313 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
3314 CGM.getDiags().Report(DiagID)
3315 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3316 return;
3317 }
3318
3319 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
3320 if (!MD)
3321 return;
3322
3323 for (llvm::MDNode *MN : MD->operands()) {
3324 auto &&GetMDInt = [MN](unsigned Idx) {
3325 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3326 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3327 };
3328
3329 auto &&GetMDString = [MN](unsigned Idx) {
3330 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
3331 return V->getString();
3332 };
3333
3334 switch (GetMDInt(0)) {
3335 default:
3336 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3336)
;
3337 break;
3338 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3339 OffloadingEntryInfoTargetRegion:
3340 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3341 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
3342 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
3343 /*Order=*/GetMDInt(5));
3344 break;
3345 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3346 OffloadingEntryInfoDeviceGlobalVar:
3347 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
3348 /*MangledName=*/GetMDString(1),
3349 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3350 /*Flags=*/GetMDInt(2)),
3351 /*Order=*/GetMDInt(3));
3352 break;
3353 }
3354 }
3355}
3356
3357void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3358 if (!KmpRoutineEntryPtrTy) {
3359 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3360 ASTContext &C = CGM.getContext();
3361 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3362 FunctionProtoType::ExtProtoInfo EPI;
3363 KmpRoutineEntryPtrQTy = C.getPointerType(
3364 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3365 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3366 }
3367}
3368
3369QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
3370 // Make sure the type of the entry is already created. This is the type we
3371 // have to create:
3372 // struct __tgt_offload_entry{
3373 // void *addr; // Pointer to the offload entry info.
3374 // // (function or global)
3375 // char *name; // Name of the function or global.
3376 // size_t size; // Size of the entry info (0 if it a function).
3377 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
3378 // int32_t reserved; // Reserved, to use by the runtime library.
3379 // };
3380 if (TgtOffloadEntryQTy.isNull()) {
3381 ASTContext &C = CGM.getContext();
3382 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
3383 RD->startDefinition();
3384 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3385 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
3386 addFieldToRecordDecl(C, RD, C.getSizeType());
3387 addFieldToRecordDecl(
3388 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3389 addFieldToRecordDecl(
3390 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3391 RD->completeDefinition();
3392 RD->addAttr(PackedAttr::CreateImplicit(C));
3393 TgtOffloadEntryQTy = C.getRecordType(RD);
3394 }
3395 return TgtOffloadEntryQTy;
3396}
3397
3398namespace {
3399struct PrivateHelpersTy {
3400 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3401 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3402 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3403 PrivateElemInit(PrivateElemInit) {}
3404 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3405 const Expr *OriginalRef = nullptr;
3406 const VarDecl *Original = nullptr;
3407 const VarDecl *PrivateCopy = nullptr;
3408 const VarDecl *PrivateElemInit = nullptr;
3409 bool isLocalPrivate() const {
3410 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3411 }
3412};
3413typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3414} // anonymous namespace
3415
3416static bool isAllocatableDecl(const VarDecl *VD) {
3417 const VarDecl *CVD = VD->getCanonicalDecl();
3418 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3419 return false;
3420 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3421 // Use the default allocation.
3422 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
3423 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
3424 !AA->getAllocator());
3425}
3426
3427static RecordDecl *
3428createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3429 if (!Privates.empty()) {
3430 ASTContext &C = CGM.getContext();
3431 // Build struct .kmp_privates_t. {
3432 // /* private vars */
3433 // };
3434 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
3435 RD->startDefinition();
3436 for (const auto &Pair : Privates) {
3437 const VarDecl *VD = Pair.second.Original;
3438 QualType Type = VD->getType().getNonReferenceType();
3439 // If the private variable is a local variable with lvalue ref type,
3440 // allocate the pointer instead of the pointee type.
3441 if (Pair.second.isLocalPrivate()) {
3442 if (VD->getType()->isLValueReferenceType())
3443 Type = C.getPointerType(Type);
3444 if (isAllocatableDecl(VD))
3445 Type = C.getPointerType(Type);
3446 }
3447 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
3448 if (VD->hasAttrs()) {
3449 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3450 E(VD->getAttrs().end());
3451 I != E; ++I)
3452 FD->addAttr(*I);
3453 }
3454 }
3455 RD->completeDefinition();
3456 return RD;
3457 }
3458 return nullptr;
3459}
3460
3461static RecordDecl *
3462createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3463 QualType KmpInt32Ty,
3464 QualType KmpRoutineEntryPointerQTy) {
3465 ASTContext &C = CGM.getContext();
3466 // Build struct kmp_task_t {
3467 // void * shareds;
3468 // kmp_routine_entry_t routine;
3469 // kmp_int32 part_id;
3470 // kmp_cmplrdata_t data1;
3471 // kmp_cmplrdata_t data2;
3472 // For taskloops additional fields:
3473 // kmp_uint64 lb;
3474 // kmp_uint64 ub;
3475 // kmp_int64 st;
3476 // kmp_int32 liter;
3477 // void * reductions;
3478 // };
3479 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
3480 UD->startDefinition();
3481 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3482 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3483 UD->completeDefinition();
3484 QualType KmpCmplrdataTy = C.getRecordType(UD);
3485 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
3486 RD->startDefinition();
3487 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3488 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3489 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3490 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3491 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3492 if (isOpenMPTaskLoopDirective(Kind)) {
3493 QualType KmpUInt64Ty =
3494 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3495 QualType KmpInt64Ty =
3496 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3497 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3498 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3499 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3500 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3501 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3502 }
3503 RD->completeDefinition();
3504 return RD;
3505}
3506
3507static RecordDecl *
3508createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3509 ArrayRef<PrivateDataTy> Privates) {
3510 ASTContext &C = CGM.getContext();
3511 // Build struct kmp_task_t_with_privates {
3512 // kmp_task_t task_data;
3513 // .kmp_privates_t. privates;
3514 // };
3515 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3516 RD->startDefinition();
3517 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
3518 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3519 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
3520 RD->completeDefinition();
3521 return RD;
3522}
3523
3524/// Emit a proxy function which accepts kmp_task_t as the second
3525/// argument.
3526/// \code
3527/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3528/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3529/// For taskloops:
3530/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3531/// tt->reductions, tt->shareds);
3532/// return 0;
3533/// }
3534/// \endcode
3535static llvm::Function *
3536emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3537 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3538 QualType KmpTaskTWithPrivatesPtrQTy,
3539 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3540 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3541 llvm::Value *TaskPrivatesMap) {
3542 ASTContext &C = CGM.getContext();
3543 FunctionArgList Args;
3544 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3545 ImplicitParamDecl::Other);
3546 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3547 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3548 ImplicitParamDecl::Other);
3549 Args.push_back(&GtidArg);
3550 Args.push_back(&TaskTypeArg);
3551 const auto &TaskEntryFnInfo =
3552 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3553 llvm::FunctionType *TaskEntryTy =
3554 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3555 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
3556 auto *TaskEntry = llvm::Function::Create(
3557 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3558 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
3559 TaskEntry->setDoesNotRecurse();
3560 CodeGenFunction CGF(CGM);
3561 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3562 Loc, Loc);
3563
3564 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3565 // tt,
3566 // For taskloops:
3567 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3568 // tt->task_data.shareds);
3569 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3570 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3571 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3572 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3573 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3574 const auto *KmpTaskTWithPrivatesQTyRD =
3575 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3576 LValue Base =
3577 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3578 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
3579 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3580 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3581 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3582
3583 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3584 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3585 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3586 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3587 CGF.ConvertTypeForMem(SharedsPtrTy));
3588
3589 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3590 llvm::Value *PrivatesParam;
3591 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3592 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3593 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3594 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3595 } else {
3596 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3597 }
3598
3599 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3600 TaskPrivatesMap,
3601 CGF.Builder
3602 .CreatePointerBitCastOrAddrSpaceCast(
3603 TDBase.getAddress(CGF), CGF.VoidPtrTy)
3604 .getPointer()};
3605 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3606 std::end(CommonArgs));
3607 if (isOpenMPTaskLoopDirective(Kind)) {
3608 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3609 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3610 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3611 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3612 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3613 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3614 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3615 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3616 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3617 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3618 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3619 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3620 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3621 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3622 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3623 CallArgs.push_back(LBParam);
3624 CallArgs.push_back(UBParam);
3625 CallArgs.push_back(StParam);
3626 CallArgs.push_back(LIParam);
3627 CallArgs.push_back(RParam);
3628 }
3629 CallArgs.push_back(SharedsParam);
3630
3631 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3632 CallArgs);
3633 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3634 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3635 CGF.FinishFunction();
3636 return TaskEntry;
3637}
3638
3639static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3640 SourceLocation Loc,
3641 QualType KmpInt32Ty,
3642 QualType KmpTaskTWithPrivatesPtrQTy,
3643 QualType KmpTaskTWithPrivatesQTy) {
3644 ASTContext &C = CGM.getContext();
3645 FunctionArgList Args;
3646 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3647 ImplicitParamDecl::Other);
3648 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3649 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3650 ImplicitParamDecl::Other);
3651 Args.push_back(&GtidArg);
3652 Args.push_back(&TaskTypeArg);
3653 const auto &DestructorFnInfo =
3654 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3655 llvm::FunctionType *DestructorFnTy =
3656 CGM.getTypes().GetFunctionType(DestructorFnInfo);
3657 std::string Name =
3658 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
3659 auto *DestructorFn =
3660 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3661 Name, &CGM.getModule());
3662 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
3663 DestructorFnInfo);
3664 DestructorFn->setDoesNotRecurse();
3665 CodeGenFunction CGF(CGM);
3666 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3667 Args, Loc, Loc);
3668
3669 LValue Base = CGF.EmitLoadOfPointerLValue(
3670 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3671 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3672 const auto *KmpTaskTWithPrivatesQTyRD =
3673 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3674 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3675 Base = CGF.EmitLValueForField(Base, *FI);
3676 for (const auto *Field :
3677 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3678 if (QualType::DestructionKind DtorKind =
3679 Field->getType().isDestructedType()) {
3680 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3681 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
3682 }
3683 }
3684 CGF.FinishFunction();
3685 return DestructorFn;
3686}
3687
3688/// Emit a privates mapping function for correct handling of private and
3689/// firstprivate variables.
3690/// \code
3691/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3692/// **noalias priv1,..., <tyn> **noalias privn) {
3693/// *priv1 = &.privates.priv1;
3694/// ...;
3695/// *privn = &.privates.privn;
3696/// }
3697/// \endcode
3698static llvm::Value *
3699emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3700 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3701 ArrayRef<PrivateDataTy> Privates) {
3702 ASTContext &C = CGM.getContext();
3703 FunctionArgList Args;
3704 ImplicitParamDecl TaskPrivatesArg(
3705 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3706 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3707 ImplicitParamDecl::Other);
3708 Args.push_back(&TaskPrivatesArg);
3709 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3710 unsigned Counter = 1;
3711 for (const Expr *E : Data.PrivateVars) {
3712 Args.push_back(ImplicitParamDecl::Create(
3713 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3714 C.getPointerType(C.getPointerType(E->getType()))
3715 .withConst()
3716 .withRestrict(),
3717 ImplicitParamDecl::Other));
3718 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3719 PrivateVarsPos[VD] = Counter;
3720 ++Counter;
3721 }
3722 for (const Expr *E : Data.FirstprivateVars) {
3723 Args.push_back(ImplicitParamDecl::Create(
3724 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3725 C.getPointerType(C.getPointerType(E->getType()))
3726 .withConst()
3727 .withRestrict(),
3728 ImplicitParamDecl::Other));
3729 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3730 PrivateVarsPos[VD] = Counter;
3731 ++Counter;
3732 }
3733 for (const Expr *E : Data.LastprivateVars) {
3734 Args.push_back(ImplicitParamDecl::Create(
3735 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3736 C.getPointerType(C.getPointerType(E->getType()))
3737 .withConst()
3738 .withRestrict(),
3739 ImplicitParamDecl::Other));
3740 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3741 PrivateVarsPos[VD] = Counter;
3742 ++Counter;
3743 }
3744 for (const VarDecl *VD : Data.PrivateLocals) {
3745 QualType Ty = VD->getType().getNonReferenceType();
3746 if (VD->getType()->isLValueReferenceType())
3747 Ty = C.getPointerType(Ty);
3748 if (isAllocatableDecl(VD))
3749 Ty = C.getPointerType(Ty);
3750 Args.push_back(ImplicitParamDecl::Create(
3751 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3752 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3753 ImplicitParamDecl::Other));
3754 PrivateVarsPos[VD] = Counter;
3755 ++Counter;
3756 }
3757 const auto &TaskPrivatesMapFnInfo =
3758 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3759 llvm::FunctionType *TaskPrivatesMapTy =
3760 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
3761 std::string Name =
3762 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
3763 auto *TaskPrivatesMap = llvm::Function::Create(
3764 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3765 &CGM.getModule());
3766 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
3767 TaskPrivatesMapFnInfo);
3768 if (CGM.getLangOpts().Optimize) {
3769 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3770 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3771 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3772 }
3773 CodeGenFunction CGF(CGM);
3774 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3775 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3776
3777 // *privi = &.privates.privi;
3778 LValue Base = CGF.EmitLoadOfPointerLValue(
3779 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3780 TaskPrivatesArg.getType()->castAs<PointerType>());
3781 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
3782 Counter = 0;
3783 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3784 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3785 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3786 LValue RefLVal =
3787 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3788 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3789 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
3790 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3791 ++Counter;
3792 }
3793 CGF.FinishFunction();
3794 return TaskPrivatesMap;
3795}
3796
3797/// Emit initialization for private variables in task-based directives.
3798static void emitPrivatesInit(CodeGenFunction &CGF,
3799 const OMPExecutableDirective &D,
3800 Address KmpTaskSharedsPtr, LValue TDBase,
3801 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3802 QualType SharedsTy, QualType SharedsPtrTy,
3803 const OMPTaskDataTy &Data,
3804 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3805 ASTContext &C = CGF.getContext();
3806 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3807 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3808 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
3809 ? OMPD_taskloop
3810 : OMPD_task;
3811 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
3812 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3813 LValue SrcBase;
3814 bool IsTargetTask =
3815 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
3816 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
3817 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3818 // PointersArray, SizesArray, and MappersArray. The original variables for
3819 // these arrays are not captured and we get their addresses explicitly.
3820 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3821 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3822 SrcBase = CGF.MakeAddrLValue(
3823 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3824 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
3825 SharedsTy);
3826 }
3827 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3828 for (const PrivateDataTy &Pair : Privates) {
3829 // Do not initialize private locals.
3830 if (Pair.second.isLocalPrivate()) {
3831 ++FI;
3832 continue;
3833 }
3834 const VarDecl *VD = Pair.second.PrivateCopy;
3835 const Expr *Init = VD->getAnyInitializer();
3836 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3837 !CGF.isTrivialInitializer(Init)))) {
3838 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3839 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3840 const VarDecl *OriginalVD = Pair.second.Original;
3841 // Check if the variable is the target-based BasePointersArray,
3842 // PointersArray, SizesArray, or MappersArray.
3843 LValue SharedRefLValue;
3844 QualType Type = PrivateLValue.getType();
3845 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3846 if (IsTargetTask && !SharedField) {
3847 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3848 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3849 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3850 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3851 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3852 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3853 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
3854 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3854, __PRETTY_FUNCTION__))
;
3855 SharedRefLValue =
3856 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3857 } else if (ForDup) {
3858 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3859 SharedRefLValue = CGF.MakeAddrLValue(
3860 Address(SharedRefLValue.getPointer(CGF),
3861 C.getDeclAlign(OriginalVD)),
3862 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3863 SharedRefLValue.getTBAAInfo());
3864 } else if (CGF.LambdaCaptureFields.count(
3865 Pair.second.Original->getCanonicalDecl()) > 0 ||
3866 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) {
3867 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3868 } else {
3869 // Processing for implicitly captured variables.
3870 InlinedOpenMPRegionRAII Region(
3871 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3872 /*HasCancel=*/false, /*NoInheritance=*/true);
3873 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3874 }
3875 if (Type->isArrayType()) {
3876 // Initialize firstprivate array.
3877 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3878 // Perform simple memcpy.
3879 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3880 } else {
3881 // Initialize firstprivate array using element-by-element
3882 // initialization.
3883 CGF.EmitOMPAggregateAssign(
3884 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
3885 Type,
3886 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3887 Address SrcElement) {
3888 // Clean up any temporaries needed by the initialization.
3889 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3890 InitScope.addPrivate(
3891 Elem, [SrcElement]() -> Address { return SrcElement; });
3892 (void)InitScope.Privatize();
3893 // Emit initialization for single element.
3894 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3895 CGF, &CapturesInfo);
3896 CGF.EmitAnyExprToMem(Init, DestElement,
3897 Init->getType().getQualifiers(),
3898 /*IsInitializer=*/false);
3899 });
3900 }
3901 } else {
3902 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3903 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
3904 return SharedRefLValue.getAddress(CGF);
3905 });
3906 (void)InitScope.Privatize();
3907 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3908 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3909 /*capturedByInit=*/false);
3910 }
3911 } else {
3912 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3913 }
3914 }
3915 ++FI;
3916 }
3917}
3918
3919/// Check if duplication function is required for taskloops.
3920static bool checkInitIsRequired(CodeGenFunction &CGF,
3921 ArrayRef<PrivateDataTy> Privates) {
3922 bool InitRequired = false;
3923 for (const PrivateDataTy &Pair : Privates) {
3924 if (Pair.second.isLocalPrivate())
3925 continue;
3926 const VarDecl *VD = Pair.second.PrivateCopy;
3927 const Expr *Init = VD->getAnyInitializer();
3928 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
3929 !CGF.isTrivialInitializer(Init));
3930 if (InitRequired)
3931 break;
3932 }
3933 return InitRequired;
3934}
3935
3936
3937/// Emit task_dup function (for initialization of
3938/// private/firstprivate/lastprivate vars and last_iter flag)
3939/// \code
3940/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3941/// lastpriv) {
3942/// // setup lastprivate flag
3943/// task_dst->last = lastpriv;
3944/// // could be constructor calls here...
3945/// }
3946/// \endcode
3947static llvm::Value *
3948emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3949 const OMPExecutableDirective &D,
3950 QualType KmpTaskTWithPrivatesPtrQTy,
3951 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3952 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3953 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3954 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3955 ASTContext &C = CGM.getContext();
3956 FunctionArgList Args;
3957 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3958 KmpTaskTWithPrivatesPtrQTy,
3959 ImplicitParamDecl::Other);
3960 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3961 KmpTaskTWithPrivatesPtrQTy,
3962 ImplicitParamDecl::Other);
3963 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
3964 ImplicitParamDecl::Other);
3965 Args.push_back(&DstArg);
3966 Args.push_back(&SrcArg);
3967 Args.push_back(&LastprivArg);
3968 const auto &TaskDupFnInfo =
3969 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3970 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
3971 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
3972 auto *TaskDup = llvm::Function::Create(
3973 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3974 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
3975 TaskDup->setDoesNotRecurse();
3976 CodeGenFunction CGF(CGM);
3977 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3978 Loc);
3979
3980 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3981 CGF.GetAddrOfLocalVar(&DstArg),
3982 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3983 // task_dst->liter = lastpriv;
3984 if (WithLastIter) {
3985 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3986 LValue Base = CGF.EmitLValueForField(
3987 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3988 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3989 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3990 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3991 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3992 }
3993
3994 // Emit initial values for private copies (if any).
3995 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3995, __PRETTY_FUNCTION__))
;
3996 Address KmpTaskSharedsPtr = Address::invalid();
3997 if (!Data.FirstprivateVars.empty()) {
3998 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3999 CGF.GetAddrOfLocalVar(&SrcArg),
4000 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4001 LValue Base = CGF.EmitLValueForField(
4002 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4003 KmpTaskSharedsPtr = Address(
4004 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4005 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4006 KmpTaskTShareds)),
4007 Loc),
4008 CGM.getNaturalTypeAlignment(SharedsTy));
4009 }
4010 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4011 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4012 CGF.FinishFunction();
4013 return TaskDup;
4014}
4015
4016/// Checks if destructor function is required to be generated.
4017/// \return true if cleanups are required, false otherwise.
4018static bool
4019checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4020 ArrayRef<PrivateDataTy> Privates) {
4021 for (const PrivateDataTy &P : Privates) {
4022 if (P.second.isLocalPrivate())
4023 continue;
4024 QualType Ty = P.second.Original->getType().getNonReferenceType();
4025 if (Ty.isDestructedType())
4026 return true;
4027 }
4028 return false;
4029}
4030
4031namespace {
4032/// Loop generator for OpenMP iterator expression.
4033class OMPIteratorGeneratorScope final
4034 : public CodeGenFunction::OMPPrivateScope {
4035 CodeGenFunction &CGF;
4036 const OMPIteratorExpr *E = nullptr;
4037 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
4038 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
4039 OMPIteratorGeneratorScope() = delete;
4040 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
4041
4042public:
4043 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
4044 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
4045 if (!E)
4046 return;
4047 SmallVector<llvm::Value *, 4> Uppers;
4048 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4049 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
4050 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
4051 addPrivate(VD, [&CGF, VD]() {
4052 return CGF.CreateMemTemp(VD->getType(), VD->getName());
4053 });
4054 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4055 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() {
4056 return CGF.CreateMemTemp(HelperData.CounterVD->getType(),
4057 "counter.addr");
4058 });
4059 }
4060 Privatize();
4061
4062 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4063 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4064 LValue CLVal =
4065 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
4066 HelperData.CounterVD->getType());
4067 // Counter = 0;
4068 CGF.EmitStoreOfScalar(
4069 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0),
4070 CLVal);
4071 CodeGenFunction::JumpDest &ContDest =
4072 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
4073 CodeGenFunction::JumpDest &ExitDest =
4074 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
4075 // N = <number-of_iterations>;
4076 llvm::Value *N = Uppers[I];
4077 // cont:
4078 // if (Counter < N) goto body; else goto exit;
4079 CGF.EmitBlock(ContDest.getBlock());
4080 auto *CVal =
4081 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
4082 llvm::Value *Cmp =
4083 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
4084 ? CGF.Builder.CreateICmpSLT(CVal, N)
4085 : CGF.Builder.CreateICmpULT(CVal, N);
4086 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
4087 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
4088 // body:
4089 CGF.EmitBlock(BodyBB);
4090 // Iteri = Begini + Counter * Stepi;
4091 CGF.EmitIgnoredExpr(HelperData.Update);
4092 }
4093 }
4094 ~OMPIteratorGeneratorScope() {
4095 if (!E)
4096 return;
4097 for (unsigned I = E->numOfIterators(); I > 0; --I) {
4098 // Counter = Counter + 1;
4099 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);
4100 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
4101 // goto cont;
4102 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
4103 // exit:
4104 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
4105 }
4106 }
4107};
4108} // namespace
4109
4110static std::pair<llvm::Value *, llvm::Value *>
4111getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
4112 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
4113 llvm::Value *Addr;
4114 if (OASE) {
4115 const Expr *Base = OASE->getBase();
4116 Addr = CGF.EmitScalarExpr(Base);
4117 } else {
4118 Addr = CGF.EmitLValue(E).getPointer(CGF);
4119 }
4120 llvm::Value *SizeVal;
4121 QualType Ty = E->getType();
4122 if (OASE) {
4123 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
4124 for (const Expr *SE : OASE->getDimensions()) {
4125 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
4126 Sz = CGF.EmitScalarConversion(
4127 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
4128 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
4129 }
4130 } else if (const auto *ASE =
4131 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
4132 LValue UpAddrLVal =
4133 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
4134 llvm::Value *UpAddr =
4135 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
4136 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
4137 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
4138 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4139 } else {
4140 SizeVal = CGF.getTypeSize(Ty);
4141 }
4142 return std::make_pair(Addr, SizeVal);
4143}
4144
4145/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4146static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
4147 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);
4148 if (KmpTaskAffinityInfoTy.isNull()) {
4149 RecordDecl *KmpAffinityInfoRD =
4150 C.buildImplicitRecord("kmp_task_affinity_info_t");
4151 KmpAffinityInfoRD->startDefinition();
4152 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());
4153 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
4154 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
4155 KmpAffinityInfoRD->completeDefinition();
4156 KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
4157 }
4158}
4159
4160CGOpenMPRuntime::TaskResultTy
4161CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4162 const OMPExecutableDirective &D,
4163 llvm::Function *TaskFunction, QualType SharedsTy,
4164 Address Shareds, const OMPTaskDataTy &Data) {
4165 ASTContext &C = CGM.getContext();
4166 llvm::SmallVector<PrivateDataTy, 4> Privates;
4167 // Aggregate privates and sort them by the alignment.
4168 const auto *I = Data.PrivateCopies.begin();
4169 for (const Expr *E : Data.PrivateVars) {
4170 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4171 Privates.emplace_back(
4172 C.getDeclAlign(VD),
4173 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4174 /*PrivateElemInit=*/nullptr));
4175 ++I;
4176 }
4177 I = Data.FirstprivateCopies.begin();
4178 const auto *IElemInitRef = Data.FirstprivateInits.begin();
4179 for (const Expr *E : Data.FirstprivateVars) {
4180 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4181 Privates.emplace_back(
4182 C.getDeclAlign(VD),
4183 PrivateHelpersTy(
4184 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4185 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4186 ++I;
4187 ++IElemInitRef;
4188 }
4189 I = Data.LastprivateCopies.begin();
4190 for (const Expr *E : Data.LastprivateVars) {
4191 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4192 Privates.emplace_back(
4193 C.getDeclAlign(VD),
4194 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4195 /*PrivateElemInit=*/nullptr));
4196 ++I;
4197 }
4198 for (const VarDecl *VD : Data.PrivateLocals) {
4199 if (isAllocatableDecl(VD))
4200 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));
4201 else
4202 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));
4203 }
4204 llvm::stable_sort(Privates,
4205 [](const PrivateDataTy &L, const PrivateDataTy &R) {
4206 return L.first > R.first;
4207 });
4208 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4209 // Build type kmp_routine_entry_t (if not built yet).
4210 emitKmpRoutineEntryT(KmpInt32Ty);
4211 // Build type kmp_task_t (if not built yet).
4212 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4213 if (SavedKmpTaskloopTQTy.isNull()) {
4214 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4215 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4216 }
4217 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4218 } else {
4219 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4222, __PRETTY_FUNCTION__))
4220 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4222, __PRETTY_FUNCTION__))
4221 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4222, __PRETTY_FUNCTION__))
4222 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4222, __PRETTY_FUNCTION__))
;
4223 if (SavedKmpTaskTQTy.isNull()) {
4224 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4225 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4226 }
4227 KmpTaskTQTy = SavedKmpTaskTQTy;
4228 }
4229 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4230 // Build particular struct kmp_task_t for the given task.
4231 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4232 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4233 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4234 QualType KmpTaskTWithPrivatesPtrQTy =
4235 C.getPointerType(KmpTaskTWithPrivatesQTy);
4236 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4237 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4238 KmpTaskTWithPrivatesTy->getPointerTo();
4239 llvm::Value *KmpTaskTWithPrivatesTySize =
4240 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4241 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4242
4243 // Emit initial values for private copies (if any).
4244 llvm::Value *TaskPrivatesMap = nullptr;
4245 llvm::Type *TaskPrivatesMapTy =
4246 std::next(TaskFunction->arg_begin(), 3)->getType();
4247 if (!Privates.empty()) {
4248 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4249 TaskPrivatesMap =
4250 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);
4251 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4252 TaskPrivatesMap, TaskPrivatesMapTy);
4253 } else {
4254 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4255 cast<llvm::PointerType>(TaskPrivatesMapTy));
4256 }
4257 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4258 // kmp_task_t *tt);
4259 llvm::Function *TaskEntry = emitProxyTaskFunction(
4260 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4261 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4262 TaskPrivatesMap);
4263
4264 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4265 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4266 // kmp_routine_entry_t *task_entry);
4267 // Task flags. Format is taken from
4268 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
4269 // description of kmp_tasking_flags struct.
4270 enum {
4271 TiedFlag = 0x1,
4272 FinalFlag = 0x2,
4273 DestructorsFlag = 0x8,
4274 PriorityFlag = 0x20,
4275 DetachableFlag = 0x40,
4276 };
4277 unsigned Flags = Data.Tied ? TiedFlag : 0;
4278 bool NeedsCleanup = false;
4279 if (!Privates.empty()) {
4280 NeedsCleanup =
4281 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4282 if (NeedsCleanup)
4283 Flags = Flags | DestructorsFlag;
4284 }
4285 if (Data.Priority.getInt())
4286 Flags = Flags | PriorityFlag;
4287 if (D.hasClausesOfKind<OMPDetachClause>())
4288 Flags = Flags | DetachableFlag;
4289 llvm::Value *TaskFlags =
4290 Data.Final.getPointer()
4291 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4292 CGF.Builder.getInt32(FinalFlag),
4293 CGF.Builder.getInt32(/*C=*/0))
4294 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4295 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4296 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4297 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4298 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4299 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4300 TaskEntry, KmpRoutineEntryPtrTy)};
4301 llvm::Value *NewTask;
4302 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4303 // Check if we have any device clause associated with the directive.
4304 const Expr *Device = nullptr;
4305 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4306 Device = C->getDevice();
4307 // Emit device ID if any otherwise use default value.
4308 llvm::Value *DeviceID;
4309 if (Device)
4310 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
4311 CGF.Int64Ty, /*isSigned=*/true);
4312 else
4313 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
4314 AllocArgs.push_back(DeviceID);
4315 NewTask = CGF.EmitRuntimeCall(
4316 OMPBuilder.getOrCreateRuntimeFunction(
4317 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4318 AllocArgs);
4319 } else {
4320 NewTask =
4321 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4322 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4323 AllocArgs);
4324 }
4325 // Emit detach clause initialization.
4326 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4327 // task_descriptor);
4328 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4329 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4330 LValue EvtLVal = CGF.EmitLValue(Evt);
4331
4332 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4333 // int gtid, kmp_task_t *task);
4334 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
4335 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
4336 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
4337 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4338 OMPBuilder.getOrCreateRuntimeFunction(
4339 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4340 {Loc, Tid, NewTask});
4341 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
4342 Evt->getExprLoc());
4343 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
4344 }
4345 // Process affinity clauses.
4346 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4347 // Process list of affinity data.
4348 ASTContext &C = CGM.getContext();
4349 Address AffinitiesArray = Address::invalid();
4350 // Calculate number of elements to form the array of affinity data.
4351 llvm::Value *NumOfElements = nullptr;
4352 unsigned NumAffinities = 0;
4353 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4354 if (const Expr *Modifier = C->getModifier()) {
4355 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4356 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4357 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4358 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4359 NumOfElements =
4360 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4361 }
4362 } else {
4363 NumAffinities += C->varlist_size();
4364 }
4365 }
4366 getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy);
4367 // Fields ids in kmp_task_affinity_info record.
4368 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4369
4370 QualType KmpTaskAffinityInfoArrayTy;
4371 if (NumOfElements) {
4372 NumOfElements = CGF.Builder.CreateNUWAdd(
4373 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
4374 OpaqueValueExpr OVE(
4375 Loc,
4376 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),
4377 VK_RValue);
4378 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4379 RValue::get(NumOfElements));
4380 KmpTaskAffinityInfoArrayTy =
4381 C.getVariableArrayType(KmpTaskAffinityInfoTy, &OVE, ArrayType::Normal,
4382 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4383 // Properly emit variable-sized array.
4384 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,
4385 ImplicitParamDecl::Other);
4386 CGF.EmitVarDecl(*PD);
4387 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
4388 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4389 /*isSigned=*/false);
4390 } else {
4391 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4392 KmpTaskAffinityInfoTy,
4393 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,
4394 ArrayType::Normal, /*IndexTypeQuals=*/0);
4395 AffinitiesArray =
4396 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
4397 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
4398 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,
4399 /*isSigned=*/false);
4400 }
4401
4402 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4403 // Fill array by elements without iterators.
4404 unsigned Pos = 0;
4405 bool HasIterator = false;
4406 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4407 if (C->getModifier()) {
4408 HasIterator = true;
4409 continue;
4410 }
4411 for (const Expr *E : C->varlists()) {
4412 llvm::Value *Addr;
4413 llvm::Value *Size;
4414 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4415 LValue Base =
4416 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
4417 KmpTaskAffinityInfoTy);
4418 // affs[i].base_addr = &<Affinities[i].second>;
4419 LValue BaseAddrLVal = CGF.EmitLValueForField(
4420 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4421 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4422 BaseAddrLVal);
4423 // affs[i].len = sizeof(<Affinities[i].second>);
4424 LValue LenLVal = CGF.EmitLValueForField(
4425 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4426 CGF.EmitStoreOfScalar(Size, LenLVal);
4427 ++Pos;
4428 }
4429 }
4430 LValue PosLVal;
4431 if (HasIterator) {
4432 PosLVal = CGF.MakeAddrLValue(
4433 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
4434 C.getSizeType());
4435 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4436 }
4437 // Process elements with iterators.
4438 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4439 const Expr *Modifier = C->getModifier();
4440 if (!Modifier)
4441 continue;
4442 OMPIteratorGeneratorScope IteratorScope(
4443 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
4444 for (const Expr *E : C->varlists()) {
4445 llvm::Value *Addr;
4446 llvm::Value *Size;
4447 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4448 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4449 LValue Base = CGF.MakeAddrLValue(
4450 Address(CGF.Builder.CreateGEP(AffinitiesArray.getPointer(), Idx),
4451 AffinitiesArray.getAlignment()),
4452 KmpTaskAffinityInfoTy);
4453 // affs[i].base_addr = &<Affinities[i].second>;
4454 LValue BaseAddrLVal = CGF.EmitLValueForField(
4455 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4456 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4457 BaseAddrLVal);
4458 // affs[i].len = sizeof(<Affinities[i].second>);
4459 LValue LenLVal = CGF.EmitLValueForField(
4460 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4461 CGF.EmitStoreOfScalar(Size, LenLVal);
4462 Idx = CGF.Builder.CreateNUWAdd(
4463 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4464 CGF.EmitStoreOfScalar(Idx, PosLVal);
4465 }
4466 }
4467 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4468 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4469 // naffins, kmp_task_affinity_info_t *affin_list);
4470 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4471 llvm::Value *GTid = getThreadID(CGF, Loc);
4472 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4473 AffinitiesArray.getPointer(), CGM.VoidPtrTy);
4474 // FIXME: Emit the function and ignore its result for now unless the
4475 // runtime function is properly implemented.
4476 (void)CGF.EmitRuntimeCall(
4477 OMPBuilder.getOrCreateRuntimeFunction(
4478 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4479 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4480 }
4481 llvm::Value *NewTaskNewTaskTTy =
4482 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4483 NewTask, KmpTaskTWithPrivatesPtrTy);
4484 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4485 KmpTaskTWithPrivatesQTy);
4486 LValue TDBase =
4487 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4488 // Fill the data in the resulting kmp_task_t record.
4489 // Copy shareds if there are any.
4490 Address KmpTaskSharedsPtr = Address::invalid();
4491 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4492 KmpTaskSharedsPtr =
4493 Address(CGF.EmitLoadOfScalar(
4494 CGF.EmitLValueForField(
4495 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4496 KmpTaskTShareds)),
4497 Loc),
4498 CGM.getNaturalTypeAlignment(SharedsTy));
4499 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4500 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4501 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4502 }
4503 // Emit initial values for private copies (if any).
4504 TaskResultTy Result;
4505 if (!Privates.empty()) {
4506 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4507 SharedsTy, SharedsPtrTy, Data, Privates,
4508 /*ForDup=*/false);
4509 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4510 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4511 Result.TaskDupFn = emitTaskDupFunction(
4512 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4513 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4514 /*WithLastIter=*/!Data.LastprivateVars.empty());
4515 }
4516 }
4517 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4518 enum { Priority = 0, Destructors = 1 };
4519 // Provide pointer to function with destructors for privates.
4520 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4521 const RecordDecl *KmpCmplrdataUD =
4522 (*FI)->getType()->getAsUnionType()->getDecl();
4523 if (NeedsCleanup) {
4524 llvm::Value *DestructorFn = emitDestructorsFunction(
4525 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4526 KmpTaskTWithPrivatesQTy);
4527 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4528 LValue DestructorsLV = CGF.EmitLValueForField(
4529 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4530 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4531 DestructorFn, KmpRoutineEntryPtrTy),
4532 DestructorsLV);
4533 }
4534 // Set priority.
4535 if (Data.Priority.getInt()) {
4536 LValue Data2LV = CGF.EmitLValueForField(
4537 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4538 LValue PriorityLV = CGF.EmitLValueForField(
4539 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4540 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4541 }
4542 Result.NewTask = NewTask;
4543 Result.TaskEntry = TaskEntry;
4544 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4545 Result.TDBase = TDBase;
4546 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4547 return Result;
4548}
4549
4550namespace {
4551/// Dependence kind for RTL.
4552enum RTLDependenceKindTy {
4553 DepIn = 0x01,
4554 DepInOut = 0x3,
4555 DepMutexInOutSet = 0x4
4556};
4557/// Fields ids in kmp_depend_info record.
4558enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4559} // namespace
4560
4561/// Translates internal dependency kind into the runtime kind.
4562static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4563 RTLDependenceKindTy DepKind;
4564 switch (K) {
4565 case OMPC_DEPEND_in:
4566 DepKind = DepIn;
4567 break;
4568 // Out and InOut dependencies must use the same code.
4569 case OMPC_DEPEND_out:
4570 case OMPC_DEPEND_inout:
4571 DepKind = DepInOut;
4572 break;
4573 case OMPC_DEPEND_mutexinoutset:
4574 DepKind = DepMutexInOutSet;
4575 break;
4576 case OMPC_DEPEND_source:
4577 case OMPC_DEPEND_sink:
4578 case OMPC_DEPEND_depobj:
4579 case OMPC_DEPEND_unknown:
4580 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4580)
;
4581 }
4582 return DepKind;
4583}
4584
4585/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4586static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4587 QualType &FlagsTy) {
4588 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4589 if (KmpDependInfoTy.isNull()) {
4590 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4591 KmpDependInfoRD->startDefinition();
4592 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4593 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4594 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4595 KmpDependInfoRD->completeDefinition();
4596 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
4597 }
4598}
4599
4600std::pair<llvm::Value *, LValue>
4601CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4602 SourceLocation Loc) {
4603 ASTContext &C = CGM.getContext();
4604 QualType FlagsTy;
4605 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4606 RecordDecl *KmpDependInfoRD =
4607 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4608 LValue Base = CGF.EmitLoadOfPointerLValue(
4609 DepobjLVal.getAddress(CGF),
4610 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4611 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4612 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4613 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4614 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4615 Base.getTBAAInfo());
4616 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4617 Addr.getPointer(),
4618 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4619 LValue NumDepsBase = CGF.MakeAddrLValue(
4620 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4621 Base.getBaseInfo(), Base.getTBAAInfo());
4622 // NumDeps = deps[i].base_addr;
4623 LValue BaseAddrLVal = CGF.EmitLValueForField(
4624 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4625 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4626 return std::make_pair(NumDeps, Base);
4627}
4628
4629static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4630 llvm::PointerUnion<unsigned *, LValue *> Pos,
4631 const OMPTaskDataTy::DependData &Data,
4632 Address DependenciesArray) {
4633 CodeGenModule &CGM = CGF.CGM;
4634 ASTContext &C = CGM.getContext();
4635 QualType FlagsTy;
4636 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4637 RecordDecl *KmpDependInfoRD =
4638 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4639 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4640
4641 OMPIteratorGeneratorScope IteratorScope(
4642 CGF, cast_or_null<OMPIteratorExpr>(
4643 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4644 : nullptr));
4645 for (const Expr *E : Data.DepExprs) {
4646 llvm::Value *Addr;
4647 llvm::Value *Size;
4648 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4649 LValue Base;
4650 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4651 Base = CGF.MakeAddrLValue(
4652 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4653 } else {
4654 LValue &PosLVal = *Pos.get<LValue *>();
4655 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4656 Base = CGF.MakeAddrLValue(
4657 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Idx),
4658 DependenciesArray.getAlignment()),
4659 KmpDependInfoTy);
4660 }
4661 // deps[i].base_addr = &<Dependencies[i].second>;
4662 LValue BaseAddrLVal = CGF.EmitLValueForField(
4663 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4664 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4665 BaseAddrLVal);
4666 // deps[i].len = sizeof(<Dependencies[i].second>);
4667 LValue LenLVal = CGF.EmitLValueForField(
4668 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
4669 CGF.EmitStoreOfScalar(Size, LenLVal);
4670 // deps[i].flags = <Dependencies[i].first>;
4671 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);
4672 LValue FlagsLVal = CGF.EmitLValueForField(
4673 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
4674 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
4675 FlagsLVal);
4676 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4677 ++(*P);
4678 } else {
4679 LValue &PosLVal = *Pos.get<LValue *>();
4680 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4681 Idx = CGF.Builder.CreateNUWAdd(Idx,
4682 llvm::ConstantInt::get(Idx->getType(), 1));
4683 CGF.EmitStoreOfScalar(Idx, PosLVal);
4684 }
4685 }
4686}
4687
4688static SmallVector<llvm::Value *, 4>
4689emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4690 const OMPTaskDataTy::DependData &Data) {
4691 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4692, __PRETTY_FUNCTION__))
4692 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4692, __PRETTY_FUNCTION__))
;
4693 SmallVector<llvm::Value *, 4> Sizes;
4694 SmallVector<LValue, 4> SizeLVals;
4695 ASTContext &C = CGF.getContext();
4696 QualType FlagsTy;
4697 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4698 RecordDecl *KmpDependInfoRD =
4699 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4700 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4701 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4702 {
4703 OMPIteratorGeneratorScope IteratorScope(
4704 CGF, cast_or_null<OMPIteratorExpr>(
4705 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4706 : nullptr));
4707 for (const Expr *E : Data.DepExprs) {
4708 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4709 LValue Base = CGF.EmitLoadOfPointerLValue(
4710 DepobjLVal.getAddress(CGF),
4711 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4712 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4713 Base.getAddress(CGF), KmpDependInfoPtrT);
4714 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4715 Base.getTBAAInfo());
4716 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4717 Addr.getPointer(),
4718 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4719 LValue NumDepsBase = CGF.MakeAddrLValue(
4720 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4721 Base.getBaseInfo(), Base.getTBAAInfo());
4722 // NumDeps = deps[i].base_addr;
4723 LValue BaseAddrLVal = CGF.EmitLValueForField(
4724 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4725 llvm::Value *NumDeps =
4726 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4727 LValue NumLVal = CGF.MakeAddrLValue(
4728 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4729 C.getUIntPtrType());
4730 CGF.InitTempAlloca(NumLVal.getAddress(CGF),
4731 llvm::ConstantInt::get(CGF.IntPtrTy, 0));
4732 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4733 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4734 CGF.EmitStoreOfScalar(Add, NumLVal);
4735 SizeLVals.push_back(NumLVal);
4736 }
4737 }
4738 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4739 llvm::Value *Size =
4740 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4741 Sizes.push_back(Size);
4742 }
4743 return Sizes;
4744}
4745
4746static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4747 LValue PosLVal,
4748 const OMPTaskDataTy::DependData &Data,
4749 Address DependenciesArray) {
4750 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4751, __PRETTY_FUNCTION__))
4751 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4751, __PRETTY_FUNCTION__))
;
4752 ASTContext &C = CGF.getContext();
4753 QualType FlagsTy;
4754 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4755 RecordDecl *KmpDependInfoRD =
4756 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4757 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4758 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4759 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4760 {
4761 OMPIteratorGeneratorScope IteratorScope(
4762 CGF, cast_or_null<OMPIteratorExpr>(
4763 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4764 : nullptr));
4765 for (unsigned I = 0, End = Data.DepExprs.size(); I < End; ++I) {
4766 const Expr *E = Data.DepExprs[I];
4767 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4768 LValue Base = CGF.EmitLoadOfPointerLValue(
4769 DepobjLVal.getAddress(CGF),
4770 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4771 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4772 Base.getAddress(CGF), KmpDependInfoPtrT);
4773 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4774 Base.getTBAAInfo());
4775
4776 // Get number of elements in a single depobj.
4777 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4778 Addr.getPointer(),
4779 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4780 LValue NumDepsBase = CGF.MakeAddrLValue(
4781 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4782 Base.getBaseInfo(), Base.getTBAAInfo());
4783 // NumDeps = deps[i].base_addr;
4784 LValue BaseAddrLVal = CGF.EmitLValueForField(
4785 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4786 llvm::Value *NumDeps =
4787 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4788
4789 // memcopy dependency data.
4790 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4791 ElSize,
4792 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4793 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4794 Address DepAddr =
4795 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Pos),
4796 DependenciesArray.getAlignment());
4797 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
4798
4799 // Increase pos.
4800 // pos += size;
4801 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4802 CGF.EmitStoreOfScalar(Add, PosLVal);
4803 }
4804 }
4805}
4806
4807std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4808 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4809 SourceLocation Loc) {
4810 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {
4811 return D.DepExprs.empty();
4812 }))
4813 return std::make_pair(nullptr, Address::invalid());
4814 // Process list of dependencies.
4815 ASTContext &C = CGM.getContext();
4816 Address DependenciesArray = Address::invalid();
4817 llvm::Value *NumOfElements = nullptr;
4818 unsigned NumDependencies = std::accumulate(
4819 Dependencies.begin(), Dependencies.end(), 0,
4820 [](unsigned V, const OMPTaskDataTy::DependData &D) {
4821 return D.DepKind == OMPC_DEPEND_depobj
4822 ? V
4823 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4824 });
4825 QualType FlagsTy;
4826 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4827 bool HasDepobjDeps = false;
4828 bool HasRegularWithIterators = false;
4829 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4830 llvm::Value *NumOfRegularWithIterators =
4831 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4832 // Calculate number of depobj dependecies and regular deps with the iterators.
4833 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4834 if (D.DepKind == OMPC_DEPEND_depobj) {
4835 SmallVector<llvm::Value *, 4> Sizes =
4836 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4837 for (llvm::Value *Size : Sizes) {
4838 NumOfDepobjElements =
4839 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4840 }
4841 HasDepobjDeps = true;
4842 continue;
4843 }
4844 // Include number of iterations, if any.
4845 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {
4846 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4847 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4848 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4849 NumOfRegularWithIterators =
4850 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz);
4851 }
4852 HasRegularWithIterators = true;
4853 continue;
4854 }
4855 }
4856
4857 QualType KmpDependInfoArrayTy;
4858 if (HasDepobjDeps || HasRegularWithIterators) {
4859 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,
4860 /*isSigned=*/false);
4861 if (HasDepobjDeps) {
4862 NumOfElements =
4863 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4864 }
4865 if (HasRegularWithIterators) {
4866 NumOfElements =
4867 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4868 }
4869 OpaqueValueExpr OVE(Loc,
4870 C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4871 VK_RValue);
4872 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4873 RValue::get(NumOfElements));
4874 KmpDependInfoArrayTy =
4875 C.getVariableArrayType(KmpDependInfoTy, &OVE, ArrayType::Normal,
4876 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4877 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4878 // Properly emit variable-sized array.
4879 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,
4880 ImplicitParamDecl::Other);
4881 CGF.EmitVarDecl(*PD);
4882 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4883 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4884 /*isSigned=*/false);
4885 } else {
4886 KmpDependInfoArrayTy = C.getConstantArrayType(
4887 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,
4888 ArrayType::Normal, /*IndexTypeQuals=*/0);
4889 DependenciesArray =
4890 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4891 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4892 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,
4893 /*isSigned=*/false);
4894 }
4895 unsigned Pos = 0;
4896 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4897 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4898 Dependencies[I].IteratorExpr)
4899 continue;
4900 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4901 DependenciesArray);
4902 }
4903 // Copy regular dependecies with iterators.
4904 LValue PosLVal = CGF.MakeAddrLValue(
4905 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4906 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4907 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4908 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4909 !Dependencies[I].IteratorExpr)
4910 continue;
4911 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4912 DependenciesArray);
4913 }
4914 // Copy final depobj arrays without iterators.
4915 if (HasDepobjDeps) {
4916 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4917 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4918 continue;
4919 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4920 DependenciesArray);
4921 }
4922 }
4923 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4924 DependenciesArray, CGF.VoidPtrTy);
4925 return std::make_pair(NumOfElements, DependenciesArray);
4926}
4927
4928Address CGOpenMPRuntime::emitDepobjDependClause(
4929 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4930 SourceLocation Loc) {
4931 if (Dependencies.DepExprs.empty())
4932 return Address::invalid();
4933 // Process list of dependencies.
4934 ASTContext &C = CGM.getContext();
4935 Address DependenciesArray = Address::invalid();
4936 unsigned NumDependencies = Dependencies.DepExprs.size();
4937 QualType FlagsTy;
4938 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4939 RecordDecl *KmpDependInfoRD =
4940 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4941
4942 llvm::Value *Size;
4943 // Define type kmp_depend_info[<Dependencies.size()>];
4944 // For depobj reserve one extra element to store the number of elements.
4945 // It is required to handle depobj(x) update(in) construct.
4946 // kmp_depend_info[<Dependencies.size()>] deps;
4947 llvm::Value *NumDepsVal;
4948 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4949 if (const auto *IE =
4950 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {
4951 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4952 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4953 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4954 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4955 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4956 }
4957 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4958 NumDepsVal);
4959 CharUnits SizeInBytes =
4960 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4961 llvm::Value *RecSize = CGM.getSize(SizeInBytes);
4962 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4963 NumDepsVal =
4964 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4965 } else {
4966 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4967 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4968 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
4969 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4970 Size = CGM.getSize(Sz.alignTo(Align));
4971 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4972 }
4973 // Need to allocate on the dynamic memory.
4974 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4975 // Use default allocator.
4976 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4977 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4978
4979 llvm::Value *Addr =
4980 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4981 CGM.getModule(), OMPRTL___kmpc_alloc),
4982 Args, ".dep.arr.addr");
4983 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4984 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo());
4985 DependenciesArray = Address(Addr, Align);
4986 // Write number of elements in the first element of array for depobj.
4987 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4988 // deps[i].base_addr = NumDependencies;
4989 LValue BaseAddrLVal = CGF.EmitLValueForField(
4990 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4991 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4992 llvm::PointerUnion<unsigned *, LValue *> Pos;
4993 unsigned Idx = 1;
4994 LValue PosLVal;
4995 if (Dependencies.IteratorExpr) {
4996 PosLVal = CGF.MakeAddrLValue(
4997 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4998 C.getSizeType());
4999 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
5000 /*IsInit=*/true);
5001 Pos = &PosLVal;
5002 } else {
5003 Pos = &Idx;
5004 }
5005 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
5006 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5007 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy);
5008 return DependenciesArray;
5009}
5010
5011void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
5012 SourceLocation Loc) {
5013 ASTContext &C = CGM.getContext();
5014 QualType FlagsTy;
5015 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5016 LValue Base = CGF.EmitLoadOfPointerLValue(
5017 DepobjLVal.getAddress(CGF),
5018 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5019 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
5020 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5021 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
5022 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
5023 Addr.getPointer(),
5024 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
5025 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
5026 CGF.VoidPtrTy);
5027 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5028 // Use default allocator.
5029 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5030 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
5031
5032 // _kmpc_free(gtid, addr, nullptr);
5033 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5034 CGM.getModule(), OMPRTL___kmpc_free),
5035 Args);
5036}
5037
5038void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
5039 OpenMPDependClauseKind NewDepKind,
5040 SourceLocation Loc) {
5041 ASTContext &C = CGM.getContext();
5042 QualType FlagsTy;
5043 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5044 RecordDecl *KmpDependInfoRD =
5045 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5046 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5047 llvm::Value *NumDeps;
5048 LValue Base;
5049 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
5050
5051 Address Begin = Base.getAddress(CGF);
5052 // Cast from pointer to array type to pointer to single element.
5053 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps);
5054 // The basic structure here is a while-do loop.
5055 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
5056 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
5057 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5058 CGF.EmitBlock(BodyBB);
5059 llvm::PHINode *ElementPHI =
5060 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
5061 ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
5062 Begin = Address(ElementPHI, Begin.getAlignment());
5063 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
5064 Base.getTBAAInfo());
5065 // deps[i].flags = NewDepKind;
5066 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
5067 LValue FlagsLVal = CGF.EmitLValueForField(
5068 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5069 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5070 FlagsLVal);
5071
5072 // Shift the address forward by one element.
5073 Address ElementNext =
5074 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
5075 ElementPHI->addIncoming(ElementNext.getPointer(),
5076 CGF.Builder.GetInsertBlock());
5077 llvm::Value *IsEmpty =
5078 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
5079 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5080 // Done.
5081 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5082}
5083
5084void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5085 const OMPExecutableDirective &D,
5086 llvm::Function *TaskFunction,
5087 QualType SharedsTy, Address Shareds,
5088 const Expr *IfCond,
5089 const OMPTaskDataTy &Data) {
5090 if (!CGF.HaveInsertPoint())
5091 return;
5092
5093 TaskResultTy Result =
5094 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5095 llvm::Value *NewTask = Result.NewTask;
5096 llvm::Function *TaskEntry = Result.TaskEntry;
5097 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5098 LValue TDBase = Result.TDBase;
5099 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5100 // Process list of dependences.
5101 Address DependenciesArray = Address::invalid();
5102 llvm::Value *NumOfElements;
5103 std::tie(NumOfElements, DependenciesArray) =
5104 emitDependClause(CGF, Data.Dependences, Loc);
5105
5106 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5107 // libcall.
5108 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5109 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5110 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5111 // list is not empty
5112 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5113 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5114 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5115 llvm::Value *DepTaskArgs[7];
5116 if (!Data.Dependences.empty()) {
5117 DepTaskArgs[0] = UpLoc;
5118 DepTaskArgs[1] = ThreadID;
5119 DepTaskArgs[2] = NewTask;
5120 DepTaskArgs[3] = NumOfElements;
5121 DepTaskArgs[4] = DependenciesArray.getPointer();
5122 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5123 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5124 }
5125 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
5126 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5127 if (!Data.Tied) {
5128 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5129 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5130 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5131 }
5132 if (!Data.Dependences.empty()) {
5133 CGF.EmitRuntimeCall(
5134 OMPBuilder.getOrCreateRuntimeFunction(
5135 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
5136 DepTaskArgs);
5137 } else {
5138 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5139 CGM.getModule(), OMPRTL___kmpc_omp_task),
5140 TaskArgs);
5141 }
5142 // Check if parent region is untied and build return for untied task;
5143 if (auto *Region =
5144 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5145 Region->emitUntiedSwitch(CGF);
5146 };
5147
5148 llvm::Value *DepWaitTaskArgs[6];
5149 if (!Data.Dependences.empty()) {
5150 DepWaitTaskArgs[0] = UpLoc;
5151 DepWaitTaskArgs[1] = ThreadID;
5152 DepWaitTaskArgs[2] = NumOfElements;
5153 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5154 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5155 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5156 }
5157 auto &M = CGM.getModule();
5158 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
5159 TaskEntry, &Data, &DepWaitTaskArgs,
5160 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5161 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5162 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5163 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5164 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5165 // is specified.
5166 if (!Data.Dependences.empty())
5167 CGF.EmitRuntimeCall(
5168 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_wait_deps),
5169 DepWaitTaskArgs);
5170 // Call proxy_task_entry(gtid, new_task);
5171 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5172 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5173 Action.Enter(CGF);
5174 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5175 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5176 OutlinedFnArgs);
5177 };
5178
5179 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5180 // kmp_task_t *new_task);
5181 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5182 // kmp_task_t *new_task);
5183 RegionCodeGenTy RCG(CodeGen);
5184 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
5185 M, OMPRTL___kmpc_omp_task_begin_if0),
5186 TaskArgs,
5187 OMPBuilder.getOrCreateRuntimeFunction(
5188 M, OMPRTL___kmpc_omp_task_complete_if0),
5189 TaskArgs);
5190 RCG.setAction(Action);
5191 RCG(CGF);
5192 };
5193
5194 if (IfCond) {
5195 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5196 } else {
5197 RegionCodeGenTy ThenRCG(ThenCodeGen);
5198 ThenRCG(CGF);
5199 }
5200}
5201
5202void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5203 const OMPLoopDirective &D,
5204 llvm::Function *TaskFunction,
5205 QualType SharedsTy, Address Shareds,
5206 const Expr *IfCond,
5207 const OMPTaskDataTy &Data) {
5208 if (!CGF.HaveInsertPoint())
5209 return;
5210 TaskResultTy Result =
5211 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5212 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5213 // libcall.
5214 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5215 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5216 // sched, kmp_uint64 grainsize, void *task_dup);
5217 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5218 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5219 llvm::Value *IfVal;
5220 if (IfCond) {
5221 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5222 /*isSigned=*/true);
5223 } else {
5224 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5225 }
5226
5227 LValue LBLVal = CGF.EmitLValueForField(
5228 Result.TDBase,
5229 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5230 const auto *LBVar =
5231 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5232 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5233 LBLVal.getQuals(),
5234 /*IsInitializer=*/true);
5235 LValue UBLVal = CGF.EmitLValueForField(
5236 Result.TDBase,
5237 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5238 const auto *UBVar =
5239 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5240 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5241 UBLVal.getQuals(),
5242 /*IsInitializer=*/true);
5243 LValue StLVal = CGF.EmitLValueForField(
5244 Result.TDBase,
5245 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5246 const auto *StVar =
5247 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5248 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5249 StLVal.getQuals(),
5250 /*IsInitializer=*/true);
5251 // Store reductions address.
5252 LValue RedLVal = CGF.EmitLValueForField(
5253 Result.TDBase,
5254 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5255 if (Data.Reductions) {
5256 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5257 } else {
5258 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5259 CGF.getContext().VoidPtrTy);
5260 }
5261 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5262 llvm::Value *TaskArgs[] = {
5263 UpLoc,
5264 ThreadID,
5265 Result.NewTask,
5266 IfVal,
5267 LBLVal.getPointer(CGF),
5268 UBLVal.getPointer(CGF),
5269 CGF.EmitLoadOfScalar(StLVal, Loc),
5270 llvm::ConstantInt::getSigned(
5271 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5272 llvm::ConstantInt::getSigned(
5273 CGF.IntTy, Data.Schedule.getPointer()
5274 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5275 : NoSchedule),
5276 Data.Schedule.getPointer()
5277 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5278 /*isSigned=*/false)
5279 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5280 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5281 Result.TaskDupFn, CGF.VoidPtrTy)
5282 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5283 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5284 CGM.getModule(), OMPRTL___kmpc_taskloop),
5285 TaskArgs);
5286}
5287
5288/// Emit reduction operation for each element of array (required for
5289/// array sections) LHS op = RHS.
5290/// \param Type Type of array.
5291/// \param LHSVar Variable on the left side of the reduction operation
5292/// (references element of array in original variable).
5293/// \param RHSVar Variable on the right side of the reduction operation
5294/// (references element of array in original variable).
5295/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5296/// RHSVar.
5297static void EmitOMPAggregateReduction(
5298 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5299 const VarDecl *RHSVar,
5300 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5301 const Expr *, const Expr *)> &RedOpGen,
5302 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5303 const Expr *UpExpr = nullptr) {
5304 // Perform element-by-element initialization.
5305 QualType ElementTy;
5306 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5307 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5308
5309 // Drill down to the base element type on both arrays.
5310 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5311 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5312
5313 llvm::Value *RHSBegin = RHSAddr.getPointer();
5314 llvm::Value *LHSBegin = LHSAddr.getPointer();
5315 // Cast from pointer to array type to pointer to single element.
5316 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5317 // The basic structure here is a while-do loop.
5318 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5319 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5320 llvm::Value *IsEmpty =
5321 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5322 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5323
5324 // Enter the loop body, making that address the current address.
5325 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5326 CGF.EmitBlock(BodyBB);
5327
5328 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5329
5330 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5331 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5332 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5333 Address RHSElementCurrent =
5334 Address(RHSElementPHI,
5335 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5336
5337 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5338 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5339 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5340 Address LHSElementCurrent =
5341 Address(LHSElementPHI,
5342 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5343
5344 // Emit copy.
5345 CodeGenFunction::OMPPrivateScope Scope(CGF);
5346 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5347 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5348 Scope.Privatize();
5349 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5350 Scope.ForceCleanup();
5351
5352 // Shift the address forward by one element.
5353 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5354 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5355 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5356 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5357 // Check whether we've reached the end.
5358 llvm::Value *Done =
5359 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5360 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5361 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5362 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5363
5364 // Done.
5365 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5366}
5367
5368/// Emit reduction combiner. If the combiner is a simple expression emit it as
5369/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5370/// UDR combiner function.
5371static void emitReductionCombiner(CodeGenFunction &CGF,
5372 const Expr *ReductionOp) {
5373 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5374 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5375 if (const auto *DRE =
5376 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5377 if (const auto *DRD =
5378 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5379 std::pair<llvm::Function *, llvm::Function *> Reduction =
5380 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5381 RValue Func = RValue::get(Reduction.first);
5382 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5383 CGF.EmitIgnoredExpr(ReductionOp);
5384 return;
5385 }
5386 CGF.EmitIgnoredExpr(ReductionOp);
5387}
5388
5389llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5390 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5391 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5392 ArrayRef<const Expr *> ReductionOps) {
5393 ASTContext &C = CGM.getContext();
5394
5395 // void reduction_func(void *LHSArg, void *RHSArg);
5396 FunctionArgList Args;
5397 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5398 ImplicitParamDecl::Other);
5399 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5400 ImplicitParamDecl::Other);
5401 Args.push_back(&LHSArg);
5402 Args.push_back(&RHSArg);
5403 const auto &CGFI =
5404 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5405 std::string Name = getName({"omp", "reduction", "reduction_func"});
5406 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5407 llvm::GlobalValue::InternalLinkage, Name,
5408 &CGM.getModule());
5409 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5410 Fn->setDoesNotRecurse();
5411 CodeGenFunction CGF(CGM);
5412 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5413
5414 // Dst = (void*[n])(LHSArg);
5415 // Src = (void*[n])(RHSArg);
5416 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5417 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5418 ArgsType), CGF.getPointerAlign());
5419 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5420 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5421 ArgsType), CGF.getPointerAlign());
5422
5423 // ...
5424 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5425 // ...
5426 CodeGenFunction::OMPPrivateScope Scope(CGF);
5427 auto IPriv = Privates.begin();
5428 unsigned Idx = 0;
5429 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5430 const auto *RHSVar =
5431 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5432 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5433 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5434 });
5435 const auto *LHSVar =
5436 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5437 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5438 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5439 });
5440 QualType PrivTy = (*IPriv)->getType();
5441 if (PrivTy->isVariablyModifiedType()) {
5442 // Get array size and emit VLA type.
5443 ++Idx;
5444 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5445 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5446 const VariableArrayType *VLA =
5447 CGF.getContext().getAsVariableArrayType(PrivTy);
5448 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5449 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5450 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5451 CGF.EmitVariablyModifiedType(PrivTy);
5452 }
5453 }
5454 Scope.Privatize();
5455 IPriv = Privates.begin();
5456 auto ILHS = LHSExprs.begin();
5457 auto IRHS = RHSExprs.begin();
5458 for (const Expr *E : ReductionOps) {
5459 if ((*IPriv)->getType()->isArrayType()) {
5460 // Emit reduction for array section.
5461 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5462 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5463 EmitOMPAggregateReduction(
5464 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5465 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5466 emitReductionCombiner(CGF, E);
5467 });
5468 } else {
5469 // Emit reduction for array subscript or single variable.
5470 emitReductionCombiner(CGF, E);
5471 }
5472 ++IPriv;
5473 ++ILHS;
5474 ++IRHS;
5475 }
5476 Scope.ForceCleanup();
5477 CGF.FinishFunction();
5478 return Fn;
5479}
5480
5481void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5482 const Expr *ReductionOp,
5483 const Expr *PrivateRef,
5484 const DeclRefExpr *LHS,
5485 const DeclRefExpr *RHS) {
5486 if (PrivateRef->getType()->isArrayType()) {
5487 // Emit reduction for array section.
5488 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5489 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5490 EmitOMPAggregateReduction(
5491 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5492 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5493 emitReductionCombiner(CGF, ReductionOp);
5494 });
5495 } else {
5496 // Emit reduction for array subscript or single variable.
5497 emitReductionCombiner(CGF, ReductionOp);
5498 }
5499}
5500
5501void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5502 ArrayRef<const Expr *> Privates,
5503 ArrayRef<const Expr *> LHSExprs,
5504 ArrayRef<const Expr *> RHSExprs,
5505 ArrayRef<const Expr *> ReductionOps,
5506 ReductionOptionsTy Options) {
5507 if (!CGF.HaveInsertPoint())
5508 return;
5509
5510 bool WithNowait = Options.WithNowait;
5511 bool SimpleReduction = Options.SimpleReduction;
5512
5513 // Next code should be emitted for reduction:
5514 //
5515 // static kmp_critical_name lock = { 0 };
5516 //
5517 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5518 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5519 // ...
5520 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5521 // *(Type<n>-1*)rhs[<n>-1]);
5522 // }
5523 //
5524 // ...
5525 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5526 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5527 // RedList, reduce_func, &<lock>)) {
5528 // case 1:
5529 // ...
5530 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5531 // ...
5532 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5533 // break;
5534 // case 2:
5535 // ...
5536 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5537 // ...
5538 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5539 // break;
5540 // default:;
5541 // }
5542 //
5543 // if SimpleReduction is true, only the next code is generated:
5544 // ...
5545 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5546 // ...
5547
5548 ASTContext &C = CGM.getContext();
5549
5550 if (SimpleReduction) {
5551 CodeGenFunction::RunCleanupsScope Scope(CGF);
5552 auto IPriv = Privates.begin();
5553 auto ILHS = LHSExprs.begin();
5554 auto IRHS = RHSExprs.begin();
5555 for (const Expr *E : ReductionOps) {
5556 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5557 cast<DeclRefExpr>(*IRHS));
5558 ++IPriv;
5559 ++ILHS;
5560 ++IRHS;
5561 }
5562 return;
5563 }
5564
5565 // 1. Build a list of reduction variables.
5566 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5567 auto Size = RHSExprs.size();
5568 for (const Expr *E : Privates) {
5569 if (E->getType()->isVariablyModifiedType())
5570 // Reserve place for array size.
5571 ++Size;
5572 }
5573 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5574 QualType ReductionArrayTy =
5575 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5576 /*IndexTypeQuals=*/0);
5577 Address ReductionList =
5578 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5579 auto IPriv = Privates.begin();
5580 unsigned Idx = 0;
5581 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5582 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5583 CGF.Builder.CreateStore(
5584 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5585 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5586 Elem);
5587 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5588 // Store array size.
5589 ++Idx;
5590 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5591 llvm::Value *Size = CGF.Builder.CreateIntCast(
5592 CGF.getVLASize(
5593 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5594 .NumElts,
5595 CGF.SizeTy, /*isSigned=*/false);
5596 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5597 Elem);
5598 }
5599 }
5600
5601 // 2. Emit reduce_func().
5602 llvm::Function *ReductionFn = emitReductionFunction(
5603 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5604 LHSExprs, RHSExprs, ReductionOps);
5605
5606 // 3. Create static kmp_critical_name lock = { 0 };
5607 std::string Name = getName({"reduction"});
5608 llvm::Value *Lock = getCriticalRegionLock(Name);
5609
5610 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5611 // RedList, reduce_func, &<lock>);
5612 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5613 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5614 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5615 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5616 ReductionList.getPointer(), CGF.VoidPtrTy);
5617 llvm::Value *Args[] = {
5618 IdentTLoc, // ident_t *<loc>
5619 ThreadId, // i32 <gtid>
5620 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5621 ReductionArrayTySize, // size_type sizeof(RedList)
5622 RL, // void *RedList
5623 ReductionFn, // void (*) (void *, void *) <reduce_func>
5624 Lock // kmp_critical_name *&<lock>
5625 };
5626 llvm::Value *Res = CGF.EmitRuntimeCall(
5627 OMPBuilder.getOrCreateRuntimeFunction(
5628 CGM.getModule(),
5629 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5630 Args);
5631
5632 // 5. Build switch(res)
5633 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5634 llvm::SwitchInst *SwInst =
5635 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5636
5637 // 6. Build case 1:
5638 // ...
5639 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5640 // ...
5641 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5642 // break;
5643 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5644 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5645 CGF.EmitBlock(Case1BB);
5646
5647 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5648 llvm::Value *EndArgs[] = {
5649 IdentTLoc, // ident_t *<loc>
5650 ThreadId, // i32 <gtid>
5651 Lock // kmp_critical_name *&<lock>
5652 };
5653 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5654 CodeGenFunction &CGF, PrePostActionTy &Action) {
5655 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5656 auto IPriv = Privates.begin();
5657 auto ILHS = LHSExprs.begin();
5658 auto IRHS = RHSExprs.begin();
5659 for (const Expr *E : ReductionOps) {
5660 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5661 cast<DeclRefExpr>(*IRHS));
5662 ++IPriv;
5663 ++ILHS;
5664 ++IRHS;
5665 }
5666 };
5667 RegionCodeGenTy RCG(CodeGen);
5668 CommonActionTy Action(
5669 nullptr, llvm::None,
5670 OMPBuilder.getOrCreateRuntimeFunction(
5671 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5672 : OMPRTL___kmpc_end_reduce),
5673 EndArgs);
5674 RCG.setAction(Action);
5675 RCG(CGF);
5676
5677 CGF.EmitBranch(DefaultBB);
5678
5679 // 7. Build case 2:
5680 // ...
5681 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5682 // ...
5683 // break;
5684 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5685 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5686 CGF.EmitBlock(Case2BB);
5687
5688 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5689 CodeGenFunction &CGF, PrePostActionTy &Action) {
5690 auto ILHS = LHSExprs.begin();
5691 auto IRHS = RHSExprs.begin();
5692 auto IPriv = Privates.begin();
5693 for (const Expr *E : ReductionOps) {
5694 const Expr *XExpr = nullptr;
5695 const Expr *EExpr = nullptr;
5696 const Expr *UpExpr = nullptr;
5697 BinaryOperatorKind BO = BO_Comma;
5698 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5699 if (BO->getOpcode() == BO_Assign) {
5700 XExpr = BO->getLHS();
5701 UpExpr = BO->getRHS();
5702 }
5703 }
5704 // Try to emit update expression as a simple atomic.
5705 const Expr *RHSExpr = UpExpr;
5706 if (RHSExpr) {
5707 // Analyze RHS part of the whole expression.
5708 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5709 RHSExpr->IgnoreParenImpCasts())) {
5710 // If this is a conditional operator, analyze its condition for
5711 // min/max reduction operator.
5712 RHSExpr = ACO->getCond();
5713 }
5714 if (const auto *BORHS =
5715 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5716 EExpr = BORHS->getRHS();
5717 BO = BORHS->getOpcode();
5718 }
5719 }
5720 if (XExpr) {
5721 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5722 auto &&AtomicRedGen = [BO, VD,
5723 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5724 const Expr *EExpr, const Expr *UpExpr) {
5725 LValue X = CGF.EmitLValue(XExpr);
5726 RValue E;
5727 if (EExpr)
5728 E = CGF.EmitAnyExpr(EExpr);
5729 CGF.EmitOMPAtomicSimpleUpdateExpr(
5730 X, E, BO, /*IsXLHSInRHSPart=*/true,
5731 llvm::AtomicOrdering::Monotonic, Loc,
5732 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5733 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5734 PrivateScope.addPrivate(
5735 VD, [&CGF, VD, XRValue, Loc]() {
5736 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5737 CGF.emitOMPSimpleStore(
5738 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5739 VD->getType().getNonReferenceType(), Loc);
5740 return LHSTemp;
5741 });
5742 (void)PrivateScope.Privatize();
5743 return CGF.EmitAnyExpr(UpExpr);
5744 });
5745 };
5746 if ((*IPriv)->getType()->isArrayType()) {
5747 // Emit atomic reduction for array section.
5748 const auto *RHSVar =
5749 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5750 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5751 AtomicRedGen, XExpr, EExpr, UpExpr);
5752 } else {
5753 // Emit atomic reduction for array subscript or single variable.
5754 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5755 }
5756 } else {
5757 // Emit as a critical region.
5758 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5759 const Expr *, const Expr *) {
5760 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5761 std::string Name = RT.getName({"atomic_reduction"});
5762 RT.emitCriticalRegion(
5763 CGF, Name,
5764 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5765 Action.Enter(CGF);
5766 emitReductionCombiner(CGF, E);
5767 },
5768 Loc);
5769 };
5770 if ((*IPriv)->getType()->isArrayType()) {
5771 const auto *LHSVar =
5772 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5773 const auto *RHSVar =
5774 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5775 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5776 CritRedGen);
5777 } else {
5778 CritRedGen(CGF, nullptr, nullptr, nullptr);
5779 }
5780 }
5781 ++ILHS;
5782 ++IRHS;
5783 ++IPriv;
5784 }
5785 };
5786 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5787 if (!WithNowait) {
5788 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5789 llvm::Value *EndArgs[] = {
5790 IdentTLoc, // ident_t *<loc>
5791 ThreadId, // i32 <gtid>
5792 Lock // kmp_critical_name *&<lock>
5793 };
5794 CommonActionTy Action(nullptr, llvm::None,
5795 OMPBuilder.getOrCreateRuntimeFunction(
5796 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5797 EndArgs);
5798 AtomicRCG.setAction(Action);
5799 AtomicRCG(CGF);
5800 } else {
5801 AtomicRCG(CGF);
5802 }
5803
5804 CGF.EmitBranch(DefaultBB);
5805 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5806}
5807
5808/// Generates unique name for artificial threadprivate variables.
5809/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5810static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5811 const Expr *Ref) {
5812 SmallString<256> Buffer;
5813 llvm::raw_svector_ostream Out(Buffer);
5814 const clang::DeclRefExpr *DE;
5815 const VarDecl *D = ::getBaseDecl(Ref, DE);
5816 if (!D)
5817 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5818 D = D-