Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 7990, column 9
2nd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/include -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-01-26-035717-31997-1 -x c++ /build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/FileManager.h"
25#include "clang/Basic/OpenMPKinds.h"
26#include "clang/Basic/SourceManager.h"
27#include "clang/CodeGen/ConstantInitBuilder.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/SetOperations.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/Bitcode/BitcodeReader.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Value.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40#include <numeric>
41
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46namespace {
47/// Base class for handling code generation inside OpenMP regions.
48class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
49public:
50 /// Kinds of OpenMP regions used in codegen.
51 enum CGOpenMPRegionKind {
52 /// Region with outlined function for standalone 'parallel'
53 /// directive.
54 ParallelOutlinedRegion,
55 /// Region with outlined function for standalone 'task' directive.
56 TaskOutlinedRegion,
57 /// Region for constructs that do not require function outlining,
58 /// like 'for', 'sections', 'atomic' etc. directives.
59 InlinedRegion,
60 /// Region with outlined function for standalone 'target' directive.
61 TargetRegion,
62 };
63
64 CGOpenMPRegionInfo(const CapturedStmt &CS,
65 const CGOpenMPRegionKind RegionKind,
66 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
67 bool HasCancel)
68 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
69 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
70
71 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
72 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
73 bool HasCancel)
74 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
75 Kind(Kind), HasCancel(HasCancel) {}
76
77 /// Get a variable or parameter for storing global thread id
78 /// inside OpenMP construct.
79 virtual const VarDecl *getThreadIDVariable() const = 0;
80
81 /// Emit the captured statement body.
82 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
83
84 /// Get an LValue for the current ThreadID variable.
85 /// \return LValue for thread id variable. This LValue always has type int32*.
86 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
87
88 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
89
90 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
91
92 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
93
94 bool hasCancel() const { return HasCancel; }
95
96 static bool classof(const CGCapturedStmtInfo *Info) {
97 return Info->getKind() == CR_OpenMP;
98 }
99
100 ~CGOpenMPRegionInfo() override = default;
101
102protected:
103 CGOpenMPRegionKind RegionKind;
104 RegionCodeGenTy CodeGen;
105 OpenMPDirectiveKind Kind;
106 bool HasCancel;
107};
108
109/// API for captured statement code generation in OpenMP constructs.
110class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
111public:
112 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
113 const RegionCodeGenTy &CodeGen,
114 OpenMPDirectiveKind Kind, bool HasCancel,
115 StringRef HelperName)
116 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
117 HasCancel),
118 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
119 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 119, __PRETTY_FUNCTION__))
;
120 }
121
122 /// Get a variable or parameter for storing global thread id
123 /// inside OpenMP construct.
124 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
125
126 /// Get the name of the capture helper.
127 StringRef getHelperName() const override { return HelperName; }
128
129 static bool classof(const CGCapturedStmtInfo *Info) {
130 return CGOpenMPRegionInfo::classof(Info) &&
131 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
132 ParallelOutlinedRegion;
133 }
134
135private:
136 /// A variable or parameter storing global thread id for OpenMP
137 /// constructs.
138 const VarDecl *ThreadIDVar;
139 StringRef HelperName;
140};
141
142/// API for captured statement code generation in OpenMP constructs.
143class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
144public:
145 class UntiedTaskActionTy final : public PrePostActionTy {
146 bool Untied;
147 const VarDecl *PartIDVar;
148 const RegionCodeGenTy UntiedCodeGen;
149 llvm::SwitchInst *UntiedSwitch = nullptr;
150
151 public:
152 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
153 const RegionCodeGenTy &UntiedCodeGen)
154 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
155 void Enter(CodeGenFunction &CGF) override {
156 if (Untied) {
157 // Emit task switching point.
158 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
159 CGF.GetAddrOfLocalVar(PartIDVar),
160 PartIDVar->getType()->castAs<PointerType>());
161 llvm::Value *Res =
162 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
163 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
164 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
165 CGF.EmitBlock(DoneBB);
166 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
167 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
168 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
169 CGF.Builder.GetInsertBlock());
170 emitUntiedSwitch(CGF);
171 }
172 }
173 void emitUntiedSwitch(CodeGenFunction &CGF) const {
174 if (Untied) {
175 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
176 CGF.GetAddrOfLocalVar(PartIDVar),
177 PartIDVar->getType()->castAs<PointerType>());
178 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
179 PartIdLVal);
180 UntiedCodeGen(CGF);
181 CodeGenFunction::JumpDest CurPoint =
182 CGF.getJumpDestInCurrentScope(".untied.next.");
183 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
184 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
185 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
186 CGF.Builder.GetInsertBlock());
187 CGF.EmitBranchThroughCleanup(CurPoint);
188 CGF.EmitBlock(CurPoint.getBlock());
189 }
190 }
191 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
192 };
193 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
194 const VarDecl *ThreadIDVar,
195 const RegionCodeGenTy &CodeGen,
196 OpenMPDirectiveKind Kind, bool HasCancel,
197 const UntiedTaskActionTy &Action)
198 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
199 ThreadIDVar(ThreadIDVar), Action(Action) {
200 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 200, __PRETTY_FUNCTION__))
;
201 }
202
203 /// Get a variable or parameter for storing global thread id
204 /// inside OpenMP construct.
205 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
206
207 /// Get an LValue for the current ThreadID variable.
208 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
209
210 /// Get the name of the capture helper.
211 StringRef getHelperName() const override { return ".omp_outlined."; }
212
213 void emitUntiedSwitch(CodeGenFunction &CGF) override {
214 Action.emitUntiedSwitch(CGF);
215 }
216
217 static bool classof(const CGCapturedStmtInfo *Info) {
218 return CGOpenMPRegionInfo::classof(Info) &&
219 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
220 TaskOutlinedRegion;
221 }
222
223private:
224 /// A variable or parameter storing global thread id for OpenMP
225 /// constructs.
226 const VarDecl *ThreadIDVar;
227 /// Action for emitting code for untied tasks.
228 const UntiedTaskActionTy &Action;
229};
230
231/// API for inlined captured statement code generation in OpenMP
232/// constructs.
233class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
234public:
235 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
236 const RegionCodeGenTy &CodeGen,
237 OpenMPDirectiveKind Kind, bool HasCancel)
238 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
239 OldCSI(OldCSI),
240 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
241
242 // Retrieve the value of the context parameter.
243 llvm::Value *getContextValue() const override {
244 if (OuterRegionInfo)
245 return OuterRegionInfo->getContextValue();
246 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 246)
;
247 }
248
249 void setContextValue(llvm::Value *V) override {
250 if (OuterRegionInfo) {
251 OuterRegionInfo->setContextValue(V);
252 return;
253 }
254 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 254)
;
255 }
256
257 /// Lookup the captured field decl for a variable.
258 const FieldDecl *lookup(const VarDecl *VD) const override {
259 if (OuterRegionInfo)
260 return OuterRegionInfo->lookup(VD);
261 // If there is no outer outlined region,no need to lookup in a list of
262 // captured variables, we can use the original one.
263 return nullptr;
264 }
265
266 FieldDecl *getThisFieldDecl() const override {
267 if (OuterRegionInfo)
268 return OuterRegionInfo->getThisFieldDecl();
269 return nullptr;
270 }
271
272 /// Get a variable or parameter for storing global thread id
273 /// inside OpenMP construct.
274 const VarDecl *getThreadIDVariable() const override {
275 if (OuterRegionInfo)
276 return OuterRegionInfo->getThreadIDVariable();
277 return nullptr;
278 }
279
280 /// Get an LValue for the current ThreadID variable.
281 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
284 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 284)
;
285 }
286
287 /// Get the name of the capture helper.
288 StringRef getHelperName() const override {
289 if (auto *OuterRegionInfo = getOldCSI())
290 return OuterRegionInfo->getHelperName();
291 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 291)
;
292 }
293
294 void emitUntiedSwitch(CodeGenFunction &CGF) override {
295 if (OuterRegionInfo)
296 OuterRegionInfo->emitUntiedSwitch(CGF);
297 }
298
299 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
300
301 static bool classof(const CGCapturedStmtInfo *Info) {
302 return CGOpenMPRegionInfo::classof(Info) &&
303 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
304 }
305
306 ~CGOpenMPInlinedRegionInfo() override = default;
307
308private:
309 /// CodeGen info about outer OpenMP region.
310 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
311 CGOpenMPRegionInfo *OuterRegionInfo;
312};
313
314/// API for captured statement code generation in OpenMP target
315/// constructs. For this captures, implicit parameters are used instead of the
316/// captured fields. The name of the target region has to be unique in a given
317/// application so it is provided by the client, because only the client has
318/// the information to generate that.
319class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
320public:
321 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
322 const RegionCodeGenTy &CodeGen, StringRef HelperName)
323 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
324 /*HasCancel=*/false),
325 HelperName(HelperName) {}
326
327 /// This is unused for target regions because each starts executing
328 /// with a single thread.
329 const VarDecl *getThreadIDVariable() const override { return nullptr; }
330
331 /// Get the name of the capture helper.
332 StringRef getHelperName() const override { return HelperName; }
333
334 static bool classof(const CGCapturedStmtInfo *Info) {
335 return CGOpenMPRegionInfo::classof(Info) &&
336 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
337 }
338
339private:
340 StringRef HelperName;
341};
342
343static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
344 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 344)
;
345}
346/// API for generation of expressions captured in a innermost OpenMP
347/// region.
348class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
349public:
350 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
351 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
352 OMPD_unknown,
353 /*HasCancel=*/false),
354 PrivScope(CGF) {
355 // Make sure the globals captured in the provided statement are local by
356 // using the privatization logic. We assume the same variable is not
357 // captured more than once.
358 for (const auto &C : CS.captures()) {
359 if (!C.capturesVariable() && !C.capturesVariableByCopy())
360 continue;
361
362 const VarDecl *VD = C.getCapturedVar();
363 if (VD->isLocalVarDeclOrParm())
364 continue;
365
366 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
367 /*RefersToEnclosingVariableOrCapture=*/false,
368 VD->getType().getNonReferenceType(), VK_LValue,
369 C.getLocation());
370 PrivScope.addPrivate(
371 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
372 }
373 (void)PrivScope.Privatize();
374 }
375
376 /// Lookup the captured field decl for a variable.
377 const FieldDecl *lookup(const VarDecl *VD) const override {
378 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
379 return FD;
380 return nullptr;
381 }
382
383 /// Emit the captured statement body.
384 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
385 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 385)
;
386 }
387
388 /// Get a variable or parameter for storing global thread id
389 /// inside OpenMP construct.
390 const VarDecl *getThreadIDVariable() const override {
391 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 391)
;
392 }
393
394 /// Get the name of the capture helper.
395 StringRef getHelperName() const override {
396 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 396)
;
397 }
398
399 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
400
401private:
402 /// Private scope to capture global variables.
403 CodeGenFunction::OMPPrivateScope PrivScope;
404};
405
406/// RAII for emitting code of OpenMP constructs.
407class InlinedOpenMPRegionRAII {
408 CodeGenFunction &CGF;
409 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
410 FieldDecl *LambdaThisCaptureField = nullptr;
411 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
412
413public:
414 /// Constructs region for combined constructs.
415 /// \param CodeGen Code generation sequence for combined directives. Includes
416 /// a list of functions used for code generation of implicitly inlined
417 /// regions.
418 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
419 OpenMPDirectiveKind Kind, bool HasCancel)
420 : CGF(CGF) {
421 // Start emission for the construct.
422 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
423 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
424 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
425 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
426 CGF.LambdaThisCaptureField = nullptr;
427 BlockInfo = CGF.BlockInfo;
428 CGF.BlockInfo = nullptr;
429 }
430
431 ~InlinedOpenMPRegionRAII() {
432 // Restore original CapturedStmtInfo only if we're done with code emission.
433 auto *OldCSI =
434 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
435 delete CGF.CapturedStmtInfo;
436 CGF.CapturedStmtInfo = OldCSI;
437 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
438 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
439 CGF.BlockInfo = BlockInfo;
440 }
441};
442
443/// Values for bit flags used in the ident_t to describe the fields.
444/// All enumeric elements are named and described in accordance with the code
445/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
446enum OpenMPLocationFlags : unsigned {
447 /// Use trampoline for internal microtask.
448 OMP_IDENT_IMD = 0x01,
449 /// Use c-style ident structure.
450 OMP_IDENT_KMPC = 0x02,
451 /// Atomic reduction option for kmpc_reduce.
452 OMP_ATOMIC_REDUCE = 0x10,
453 /// Explicit 'barrier' directive.
454 OMP_IDENT_BARRIER_EXPL = 0x20,
455 /// Implicit barrier in code.
456 OMP_IDENT_BARRIER_IMPL = 0x40,
457 /// Implicit barrier in 'for' directive.
458 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
459 /// Implicit barrier in 'sections' directive.
460 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
461 /// Implicit barrier in 'single' directive.
462 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
463 /// Call of __kmp_for_static_init for static loop.
464 OMP_IDENT_WORK_LOOP = 0x200,
465 /// Call of __kmp_for_static_init for sections.
466 OMP_IDENT_WORK_SECTIONS = 0x400,
467 /// Call of __kmp_for_static_init for distribute.
468 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
469 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
470};
471
472namespace {
473LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
474/// Values for bit flags for marking which requires clauses have been used.
475enum OpenMPOffloadingRequiresDirFlags : int64_t {
476 /// flag undefined.
477 OMP_REQ_UNDEFINED = 0x000,
478 /// no requires clause present.
479 OMP_REQ_NONE = 0x001,
480 /// reverse_offload clause.
481 OMP_REQ_REVERSE_OFFLOAD = 0x002,
482 /// unified_address clause.
483 OMP_REQ_UNIFIED_ADDRESS = 0x004,
484 /// unified_shared_memory clause.
485 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
486 /// dynamic_allocators clause.
487 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
488 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
489};
490
491enum OpenMPOffloadingReservedDeviceIDs {
492 /// Device ID if the device was not defined, runtime should get it
493 /// from environment variables in the spec.
494 OMP_DEVICEID_UNDEF = -1,
495};
496} // anonymous namespace
497
498/// Describes ident structure that describes a source location.
499/// All descriptions are taken from
500/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
501/// Original structure:
502/// typedef struct ident {
503/// kmp_int32 reserved_1; /**< might be used in Fortran;
504/// see above */
505/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
506/// KMP_IDENT_KMPC identifies this union
507/// member */
508/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
509/// see above */
510///#if USE_ITT_BUILD
511/// /* but currently used for storing
512/// region-specific ITT */
513/// /* contextual information. */
514///#endif /* USE_ITT_BUILD */
515/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
516/// C++ */
517/// char const *psource; /**< String describing the source location.
518/// The string is composed of semi-colon separated
519// fields which describe the source file,
520/// the function and a pair of line numbers that
521/// delimit the construct.
522/// */
523/// } ident_t;
524enum IdentFieldIndex {
525 /// might be used in Fortran
526 IdentField_Reserved_1,
527 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
528 IdentField_Flags,
529 /// Not really used in Fortran any more
530 IdentField_Reserved_2,
531 /// Source[4] in Fortran, do not use for C++
532 IdentField_Reserved_3,
533 /// String describing the source location. The string is composed of
534 /// semi-colon separated fields which describe the source file, the function
535 /// and a pair of line numbers that delimit the construct.
536 IdentField_PSource
537};
538
539/// Schedule types for 'omp for' loops (these enumerators are taken from
540/// the enum sched_type in kmp.h).
541enum OpenMPSchedType {
542 /// Lower bound for default (unordered) versions.
543 OMP_sch_lower = 32,
544 OMP_sch_static_chunked = 33,
545 OMP_sch_static = 34,
546 OMP_sch_dynamic_chunked = 35,
547 OMP_sch_guided_chunked = 36,
548 OMP_sch_runtime = 37,
549 OMP_sch_auto = 38,
550 /// static with chunk adjustment (e.g., simd)
551 OMP_sch_static_balanced_chunked = 45,
552 /// Lower bound for 'ordered' versions.
553 OMP_ord_lower = 64,
554 OMP_ord_static_chunked = 65,
555 OMP_ord_static = 66,
556 OMP_ord_dynamic_chunked = 67,
557 OMP_ord_guided_chunked = 68,
558 OMP_ord_runtime = 69,
559 OMP_ord_auto = 70,
560 OMP_sch_default = OMP_sch_static,
561 /// dist_schedule types
562 OMP_dist_sch_static_chunked = 91,
563 OMP_dist_sch_static = 92,
564 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
565 /// Set if the monotonic schedule modifier was present.
566 OMP_sch_modifier_monotonic = (1 << 29),
567 /// Set if the nonmonotonic schedule modifier was present.
568 OMP_sch_modifier_nonmonotonic = (1 << 30),
569};
570
571/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
572/// region.
573class CleanupTy final : public EHScopeStack::Cleanup {
574 PrePostActionTy *Action;
575
576public:
577 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
578 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
579 if (!CGF.HaveInsertPoint())
580 return;
581 Action->Exit(CGF);
582 }
583};
584
585} // anonymous namespace
586
587void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
588 CodeGenFunction::RunCleanupsScope Scope(CGF);
589 if (PrePostAction) {
590 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
591 Callback(CodeGen, CGF, *PrePostAction);
592 } else {
593 PrePostActionTy Action;
594 Callback(CodeGen, CGF, Action);
595 }
596}
597
598/// Check if the combiner is a call to UDR combiner and if it is so return the
599/// UDR decl used for reduction.
600static const OMPDeclareReductionDecl *
601getReductionInit(const Expr *ReductionOp) {
602 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
603 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
604 if (const auto *DRE =
605 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
606 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
607 return DRD;
608 return nullptr;
609}
610
611static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
612 const OMPDeclareReductionDecl *DRD,
613 const Expr *InitOp,
614 Address Private, Address Original,
615 QualType Ty) {
616 if (DRD->getInitializer()) {
617 std::pair<llvm::Function *, llvm::Function *> Reduction =
618 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
619 const auto *CE = cast<CallExpr>(InitOp);
620 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
621 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
622 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
623 const auto *LHSDRE =
624 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
625 const auto *RHSDRE =
626 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
627 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
628 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
629 [=]() { return Private; });
630 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
631 [=]() { return Original; });
632 (void)PrivateScope.Privatize();
633 RValue Func = RValue::get(Reduction.second);
634 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
635 CGF.EmitIgnoredExpr(InitOp);
636 } else {
637 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
638 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
639 auto *GV = new llvm::GlobalVariable(
640 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
641 llvm::GlobalValue::PrivateLinkage, Init, Name);
642 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
643 RValue InitRVal;
644 switch (CGF.getEvaluationKind(Ty)) {
645 case TEK_Scalar:
646 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
647 break;
648 case TEK_Complex:
649 InitRVal =
650 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
651 break;
652 case TEK_Aggregate:
653 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
654 break;
655 }
656 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
657 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
658 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
659 /*IsInitializer=*/false);
660 }
661}
662
663/// Emit initialization of arrays of complex types.
664/// \param DestAddr Address of the array.
665/// \param Type Type of array.
666/// \param Init Initial expression of array.
667/// \param SrcAddr Address of the original array.
668static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
669 QualType Type, bool EmitDeclareReductionInit,
670 const Expr *Init,
671 const OMPDeclareReductionDecl *DRD,
672 Address SrcAddr = Address::invalid()) {
673 // Perform element-by-element initialization.
674 QualType ElementTy;
675
676 // Drill down to the base element type on both arrays.
677 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
678 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
679 DestAddr =
680 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
681 if (DRD)
682 SrcAddr =
683 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
684
685 llvm::Value *SrcBegin = nullptr;
686 if (DRD)
687 SrcBegin = SrcAddr.getPointer();
688 llvm::Value *DestBegin = DestAddr.getPointer();
689 // Cast from pointer to array type to pointer to single element.
690 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
691 // The basic structure here is a while-do loop.
692 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
693 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
694 llvm::Value *IsEmpty =
695 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
696 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
697
698 // Enter the loop body, making that address the current address.
699 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
700 CGF.EmitBlock(BodyBB);
701
702 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
703
704 llvm::PHINode *SrcElementPHI = nullptr;
705 Address SrcElementCurrent = Address::invalid();
706 if (DRD) {
707 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
708 "omp.arraycpy.srcElementPast");
709 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
710 SrcElementCurrent =
711 Address(SrcElementPHI,
712 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
713 }
714 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
715 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
716 DestElementPHI->addIncoming(DestBegin, EntryBB);
717 Address DestElementCurrent =
718 Address(DestElementPHI,
719 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
720
721 // Emit copy.
722 {
723 CodeGenFunction::RunCleanupsScope InitScope(CGF);
724 if (EmitDeclareReductionInit) {
725 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
726 SrcElementCurrent, ElementTy);
727 } else
728 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
729 /*IsInitializer=*/false);
730 }
731
732 if (DRD) {
733 // Shift the address forward by one element.
734 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
735 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
736 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
737 }
738
739 // Shift the address forward by one element.
740 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
741 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
742 // Check whether we've reached the end.
743 llvm::Value *Done =
744 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
745 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
746 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
747
748 // Done.
749 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
750}
751
752LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
753 return CGF.EmitOMPSharedLValue(E);
754}
755
756LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
757 const Expr *E) {
758 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
759 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
760 return LValue();
761}
762
763void ReductionCodeGen::emitAggregateInitialization(
764 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
765 const OMPDeclareReductionDecl *DRD) {
766 // Emit VarDecl with copy init for arrays.
767 // Get the address of the original variable captured in current
768 // captured region.
769 const auto *PrivateVD =
770 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
771 bool EmitDeclareReductionInit =
772 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
773 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
774 EmitDeclareReductionInit,
775 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
776 : PrivateVD->getInit(),
777 DRD, SharedLVal.getAddress(CGF));
778}
779
780ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
781 ArrayRef<const Expr *> Origs,
782 ArrayRef<const Expr *> Privates,
783 ArrayRef<const Expr *> ReductionOps) {
784 ClausesData.reserve(Shareds.size());
785 SharedAddresses.reserve(Shareds.size());
786 Sizes.reserve(Shareds.size());
787 BaseDecls.reserve(Shareds.size());
788 const auto *IOrig = Origs.begin();
789 const auto *IPriv = Privates.begin();
790 const auto *IRed = ReductionOps.begin();
791 for (const Expr *Ref : Shareds) {
792 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
793 std::advance(IOrig, 1);
794 std::advance(IPriv, 1);
795 std::advance(IRed, 1);
796 }
797}
798
799void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
800 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
801 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
;
802 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
803 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
804 SharedAddresses.emplace_back(First, Second);
805 if (ClausesData[N].Shared == ClausesData[N].Ref) {
806 OrigAddresses.emplace_back(First, Second);
807 } else {
808 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
809 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
810 OrigAddresses.emplace_back(First, Second);
811 }
812}
813
814void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
815 const auto *PrivateVD =
816 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
817 QualType PrivateType = PrivateVD->getType();
818 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
819 if (!PrivateType->isVariablyModifiedType()) {
820 Sizes.emplace_back(
821 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
822 nullptr);
823 return;
824 }
825 llvm::Value *Size;
826 llvm::Value *SizeInChars;
827 auto *ElemType =
828 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType())
829 ->getElementType();
830 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
831 if (AsArraySection) {
832 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
833 OrigAddresses[N].first.getPointer(CGF));
834 Size = CGF.Builder.CreateNUWAdd(
835 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
836 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
837 } else {
838 SizeInChars =
839 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
840 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
841 }
842 Sizes.emplace_back(SizeInChars, Size);
843 CodeGenFunction::OpaqueValueMapping OpaqueMap(
844 CGF,
845 cast<OpaqueValueExpr>(
846 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
847 RValue::get(Size));
848 CGF.EmitVariablyModifiedType(PrivateType);
849}
850
851void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
852 llvm::Value *Size) {
853 const auto *PrivateVD =
854 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
855 QualType PrivateType = PrivateVD->getType();
856 if (!PrivateType->isVariablyModifiedType()) {
857 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
858 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
859 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
;
860 return;
861 }
862 CodeGenFunction::OpaqueValueMapping OpaqueMap(
863 CGF,
864 cast<OpaqueValueExpr>(
865 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
866 RValue::get(Size));
867 CGF.EmitVariablyModifiedType(PrivateType);
868}
869
870void ReductionCodeGen::emitInitialization(
871 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
872 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
873 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 873, __PRETTY_FUNCTION__))
;
874 const auto *PrivateVD =
875 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
876 const OMPDeclareReductionDecl *DRD =
877 getReductionInit(ClausesData[N].ReductionOp);
878 QualType PrivateType = PrivateVD->getType();
879 PrivateAddr = CGF.Builder.CreateElementBitCast(
880 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
881 QualType SharedType = SharedAddresses[N].first.getType();
882 SharedLVal = CGF.MakeAddrLValue(
883 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
884 CGF.ConvertTypeForMem(SharedType)),
885 SharedType, SharedAddresses[N].first.getBaseInfo(),
886 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
887 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
888 if (DRD && DRD->getInitializer())
889 (void)DefaultInit(CGF);
890 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
891 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
892 (void)DefaultInit(CGF);
893 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
894 PrivateAddr, SharedLVal.getAddress(CGF),
895 SharedLVal.getType());
896 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
897 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
898 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
899 PrivateVD->getType().getQualifiers(),
900 /*IsInitializer=*/false);
901 }
902}
903
904bool ReductionCodeGen::needCleanups(unsigned N) {
905 const auto *PrivateVD =
906 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
907 QualType PrivateType = PrivateVD->getType();
908 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
909 return DTorKind != QualType::DK_none;
910}
911
912void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
913 Address PrivateAddr) {
914 const auto *PrivateVD =
915 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
916 QualType PrivateType = PrivateVD->getType();
917 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
918 if (needCleanups(N)) {
919 PrivateAddr = CGF.Builder.CreateElementBitCast(
920 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
921 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
922 }
923}
924
925static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
926 LValue BaseLV) {
927 BaseTy = BaseTy.getNonReferenceType();
928 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
929 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
930 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
931 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
932 } else {
933 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
934 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
935 }
936 BaseTy = BaseTy->getPointeeType();
937 }
938 return CGF.MakeAddrLValue(
939 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
940 CGF.ConvertTypeForMem(ElTy)),
941 BaseLV.getType(), BaseLV.getBaseInfo(),
942 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
943}
944
945static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
946 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
947 llvm::Value *Addr) {
948 Address Tmp = Address::invalid();
949 Address TopTmp = Address::invalid();
950 Address MostTopTmp = Address::invalid();
951 BaseTy = BaseTy.getNonReferenceType();
952 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
953 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
954 Tmp = CGF.CreateMemTemp(BaseTy);
955 if (TopTmp.isValid())
956 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
957 else
958 MostTopTmp = Tmp;
959 TopTmp = Tmp;
960 BaseTy = BaseTy->getPointeeType();
961 }
962 llvm::Type *Ty = BaseLVType;
963 if (Tmp.isValid())
964 Ty = Tmp.getElementType();
965 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
966 if (Tmp.isValid()) {
967 CGF.Builder.CreateStore(Addr, Tmp);
968 return MostTopTmp;
969 }
970 return Address(Addr, BaseLVAlignment);
971}
972
973static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
974 const VarDecl *OrigVD = nullptr;
975 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
976 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
977 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
978 Base = TempOASE->getBase()->IgnoreParenImpCasts();
979 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
980 Base = TempASE->getBase()->IgnoreParenImpCasts();
981 DE = cast<DeclRefExpr>(Base);
982 OrigVD = cast<VarDecl>(DE->getDecl());
983 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
984 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
985 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
986 Base = TempASE->getBase()->IgnoreParenImpCasts();
987 DE = cast<DeclRefExpr>(Base);
988 OrigVD = cast<VarDecl>(DE->getDecl());
989 }
990 return OrigVD;
991}
992
993Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
994 Address PrivateAddr) {
995 const DeclRefExpr *DE;
996 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
997 BaseDecls.emplace_back(OrigVD);
998 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
999 LValue BaseLValue =
1000 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1001 OriginalBaseLValue);
1002 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1003 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1004 llvm::Value *PrivatePointer =
1005 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1006 PrivateAddr.getPointer(),
1007 SharedAddresses[N].first.getAddress(CGF).getType());
1008 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1009 return castToBase(CGF, OrigVD->getType(),
1010 SharedAddresses[N].first.getType(),
1011 OriginalBaseLValue.getAddress(CGF).getType(),
1012 OriginalBaseLValue.getAlignment(), Ptr);
1013 }
1014 BaseDecls.emplace_back(
1015 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1016 return PrivateAddr;
1017}
1018
1019bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1020 const OMPDeclareReductionDecl *DRD =
1021 getReductionInit(ClausesData[N].ReductionOp);
1022 return DRD && DRD->getInitializer();
1023}
1024
1025LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1026 return CGF.EmitLoadOfPointerLValue(
1027 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1028 getThreadIDVariable()->getType()->castAs<PointerType>());
1029}
1030
1031void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1032 if (!CGF.HaveInsertPoint())
1033 return;
1034 // 1.2.2 OpenMP Language Terminology
1035 // Structured block - An executable statement with a single entry at the
1036 // top and a single exit at the bottom.
1037 // The point of exit cannot be a branch out of the structured block.
1038 // longjmp() and throw() must not violate the entry/exit criteria.
1039 CGF.EHStack.pushTerminate();
1040 CodeGen(CGF);
1041 CGF.EHStack.popTerminate();
1042}
1043
1044LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1045 CodeGenFunction &CGF) {
1046 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1047 getThreadIDVariable()->getType(),
1048 AlignmentSource::Decl);
1049}
1050
1051static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1052 QualType FieldTy) {
1053 auto *Field = FieldDecl::Create(
1054 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1055 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1056 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1057 Field->setAccess(AS_public);
1058 DC->addDecl(Field);
1059 return Field;
1060}
1061
1062CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1063 StringRef Separator)
1064 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1065 OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
1066 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1067
1068 // Initialize Types used in OpenMPIRBuilder from OMPKinds.def
1069 OMPBuilder.initialize();
1070 loadOffloadInfoMetadata();
1071}
1072
1073void CGOpenMPRuntime::clear() {
1074 InternalVars.clear();
1075 // Clean non-target variable declarations possibly used only in debug info.
1076 for (const auto &Data : EmittedNonTargetVariables) {
1077 if (!Data.getValue().pointsToAliveValue())
1078 continue;
1079 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1080 if (!GV)
1081 continue;
1082 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1083 continue;
1084 GV->eraseFromParent();
1085 }
1086}
1087
1088std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1089 SmallString<128> Buffer;
1090 llvm::raw_svector_ostream OS(Buffer);
1091 StringRef Sep = FirstSeparator;
1092 for (StringRef Part : Parts) {
1093 OS << Sep << Part;
1094 Sep = Separator;
1095 }
1096 return std::string(OS.str());
1097}
1098
1099static llvm::Function *
1100emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1101 const Expr *CombinerInitializer, const VarDecl *In,
1102 const VarDecl *Out, bool IsCombiner) {
1103 // void .omp_combiner.(Ty *in, Ty *out);
1104 ASTContext &C = CGM.getContext();
1105 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1106 FunctionArgList Args;
1107 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1108 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1109 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1110 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1111 Args.push_back(&OmpOutParm);
1112 Args.push_back(&OmpInParm);
1113 const CGFunctionInfo &FnInfo =
1114 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1115 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1116 std::string Name = CGM.getOpenMPRuntime().getName(
1117 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1118 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1119 Name, &CGM.getModule());
1120 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1121 if (CGM.getLangOpts().Optimize) {
1122 Fn->removeFnAttr(llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1125 }
1126 CodeGenFunction CGF(CGM);
1127 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1128 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1129 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1130 Out->getLocation());
1131 CodeGenFunction::OMPPrivateScope Scope(CGF);
1132 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1133 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1134 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1135 .getAddress(CGF);
1136 });
1137 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1138 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1139 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1140 .getAddress(CGF);
1141 });
1142 (void)Scope.Privatize();
1143 if (!IsCombiner && Out->hasInit() &&
1144 !CGF.isTrivialInitializer(Out->getInit())) {
1145 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1146 Out->getType().getQualifiers(),
1147 /*IsInitializer=*/true);
1148 }
1149 if (CombinerInitializer)
1150 CGF.EmitIgnoredExpr(CombinerInitializer);
1151 Scope.ForceCleanup();
1152 CGF.FinishFunction();
1153 return Fn;
1154}
1155
1156void CGOpenMPRuntime::emitUserDefinedReduction(
1157 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1158 if (UDRMap.count(D) > 0)
1159 return;
1160 llvm::Function *Combiner = emitCombinerOrInitializer(
1161 CGM, D->getType(), D->getCombiner(),
1162 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1163 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1164 /*IsCombiner=*/true);
1165 llvm::Function *Initializer = nullptr;
1166 if (const Expr *Init = D->getInitializer()) {
1167 Initializer = emitCombinerOrInitializer(
1168 CGM, D->getType(),
1169 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1170 : nullptr,
1171 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1172 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1173 /*IsCombiner=*/false);
1174 }
1175 UDRMap.try_emplace(D, Combiner, Initializer);
1176 if (CGF) {
1177 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1178 Decls.second.push_back(D);
1179 }
1180}
1181
1182std::pair<llvm::Function *, llvm::Function *>
1183CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1184 auto I = UDRMap.find(D);
1185 if (I != UDRMap.end())
1186 return I->second;
1187 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1188 return UDRMap.lookup(D);
1189}
1190
1191namespace {
1192// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1193// Builder if one is present.
1194struct PushAndPopStackRAII {
1195 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1196 bool HasCancel)
1197 : OMPBuilder(OMPBuilder) {
1198 if (!OMPBuilder)
1199 return;
1200
1201 // The following callback is the crucial part of clangs cleanup process.
1202 //
1203 // NOTE:
1204 // Once the OpenMPIRBuilder is used to create parallel regions (and
1205 // similar), the cancellation destination (Dest below) is determined via
1206 // IP. That means if we have variables to finalize we split the block at IP,
1207 // use the new block (=BB) as destination to build a JumpDest (via
1208 // getJumpDestInCurrentScope(BB)) which then is fed to
1209 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1210 // to push & pop an FinalizationInfo object.
1211 // The FiniCB will still be needed but at the point where the
1212 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1213 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1214 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
1215 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
;
1216 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1217 CGF.Builder.restoreIP(IP);
1218 CodeGenFunction::JumpDest Dest =
1219 CGF.getOMPCancelDestination(OMPD_parallel);
1220 CGF.EmitBranchThroughCleanup(Dest);
1221 };
1222
1223 // TODO: Remove this once we emit parallel regions through the
1224 // OpenMPIRBuilder as it can do this setup internally.
1225 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1226 {FiniCB, OMPD_parallel, HasCancel});
1227 OMPBuilder->pushFinalizationCB(std::move(FI));
1228 }
1229 ~PushAndPopStackRAII() {
1230 if (OMPBuilder)
1231 OMPBuilder->popFinalizationCB();
1232 }
1233 llvm::OpenMPIRBuilder *OMPBuilder;
1234};
1235} // namespace
1236
1237static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1238 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1239 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1240 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1241 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
1242 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
;
1243 CodeGenFunction CGF(CGM, true);
1244 bool HasCancel = false;
1245 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1246 HasCancel = OPD->hasCancel();
1247 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1248 HasCancel = OPD->hasCancel();
1249 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1250 HasCancel = OPSD->hasCancel();
1251 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1254 HasCancel = OPFD->hasCancel();
1255 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1256 HasCancel = OPFD->hasCancel();
1257 else if (const auto *OPFD =
1258 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1259 HasCancel = OPFD->hasCancel();
1260 else if (const auto *OPFD =
1261 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1262 HasCancel = OPFD->hasCancel();
1263
1264 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1265 // parallel region to make cancellation barriers work properly.
1266 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1267 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel);
1268 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1269 HasCancel, OutlinedHelperName);
1270 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1271 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1272}
1273
1274llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1275 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1276 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1277 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1278 return emitParallelOrTeamsOutlinedFunction(
1279 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1280}
1281
1282llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1283 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1284 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1285 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1286 return emitParallelOrTeamsOutlinedFunction(
1287 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1288}
1289
1290llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1291 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1292 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1293 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1294 bool Tied, unsigned &NumberOfParts) {
1295 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1296 PrePostActionTy &) {
1297 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1298 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1299 llvm::Value *TaskArgs[] = {
1300 UpLoc, ThreadID,
1301 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1302 TaskTVar->getType()->castAs<PointerType>())
1303 .getPointer(CGF)};
1304 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1305 CGM.getModule(), OMPRTL___kmpc_omp_task),
1306 TaskArgs);
1307 };
1308 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1309 UntiedCodeGen);
1310 CodeGen.setAction(Action);
1311 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
1312 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
;
1313 const OpenMPDirectiveKind Region =
1314 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1315 : OMPD_task;
1316 const CapturedStmt *CS = D.getCapturedStmt(Region);
1317 bool HasCancel = false;
1318 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1319 HasCancel = TD->hasCancel();
1320 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1321 HasCancel = TD->hasCancel();
1322 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1323 HasCancel = TD->hasCancel();
1324 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1325 HasCancel = TD->hasCancel();
1326
1327 CodeGenFunction CGF(CGM, true);
1328 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1329 InnermostKind, HasCancel, Action);
1330 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1331 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1332 if (!Tied)
1333 NumberOfParts = Action.getNumberOfParts();
1334 return Res;
1335}
1336
1337static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1338 const RecordDecl *RD, const CGRecordLayout &RL,
1339 ArrayRef<llvm::Constant *> Data) {
1340 llvm::StructType *StructTy = RL.getLLVMType();
1341 unsigned PrevIdx = 0;
1342 ConstantInitBuilder CIBuilder(CGM);
1343 auto DI = Data.begin();
1344 for (const FieldDecl *FD : RD->fields()) {
1345 unsigned Idx = RL.getLLVMFieldNo(FD);
1346 // Fill the alignment.
1347 for (unsigned I = PrevIdx; I < Idx; ++I)
1348 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1349 PrevIdx = Idx + 1;
1350 Fields.add(*DI);
1351 ++DI;
1352 }
1353}
1354
1355template <class... As>
1356static llvm::GlobalVariable *
1357createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1358 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1359 As &&... Args) {
1360 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1361 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1362 ConstantInitBuilder CIBuilder(CGM);
1363 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1364 buildStructValue(Fields, CGM, RD, RL, Data);
1365 return Fields.finishAndCreateGlobal(
1366 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1367 std::forward<As>(Args)...);
1368}
1369
1370template <typename T>
1371static void
1372createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1373 ArrayRef<llvm::Constant *> Data,
1374 T &Parent) {
1375 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1376 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1377 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1378 buildStructValue(Fields, CGM, RD, RL, Data);
1379 Fields.finishAndAddTo(Parent);
1380}
1381
1382void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1383 bool AtCurrentPoint) {
1384 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1385 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1385, __PRETTY_FUNCTION__))
;
1386
1387 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1388 if (AtCurrentPoint) {
1389 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1390 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1391 } else {
1392 Elem.second.ServiceInsertPt =
1393 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1394 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1395 }
1396}
1397
1398void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1399 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1400 if (Elem.second.ServiceInsertPt) {
1401 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1402 Elem.second.ServiceInsertPt = nullptr;
1403 Ptr->eraseFromParent();
1404 }
1405}
1406
1407static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1408 SourceLocation Loc,
1409 SmallString<128> &Buffer) {
1410 llvm::raw_svector_ostream OS(Buffer);
1411 // Build debug location
1412 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1413 OS << ";" << PLoc.getFilename() << ";";
1414 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1415 OS << FD->getQualifiedNameAsString();
1416 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1417 return OS.str();
1418}
1419
1420llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1421 SourceLocation Loc,
1422 unsigned Flags) {
1423 llvm::Constant *SrcLocStr;
1424 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1425 Loc.isInvalid()) {
1426 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr();
1427 } else {
1428 std::string FunctionName = "";
1429 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1430 FunctionName = FD->getQualifiedNameAsString();
1431 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1432 const char *FileName = PLoc.getFilename();
1433 unsigned Line = PLoc.getLine();
1434 unsigned Column = PLoc.getColumn();
1435 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName.c_str(), FileName,
1436 Line, Column);
1437 }
1438 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1439 return OMPBuilder.getOrCreateIdent(SrcLocStr, llvm::omp::IdentFlag(Flags),
1440 Reserved2Flags);
1441}
1442
1443llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1444 SourceLocation Loc) {
1445 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1445, __PRETTY_FUNCTION__))
;
1446 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1447 // the clang invariants used below might be broken.
1448 if (CGM.getLangOpts().OpenMPIRBuilder) {
1449 SmallString<128> Buffer;
1450 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1451 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1452 getIdentStringFromSourceLocation(CGF, Loc, Buffer));
1453 return OMPBuilder.getOrCreateThreadID(
1454 OMPBuilder.getOrCreateIdent(SrcLocStr));
1455 }
1456
1457 llvm::Value *ThreadID = nullptr;
1458 // Check whether we've already cached a load of the thread id in this
1459 // function.
1460 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1461 if (I != OpenMPLocThreadIDMap.end()) {
1462 ThreadID = I->second.ThreadID;
1463 if (ThreadID != nullptr)
1464 return ThreadID;
1465 }
1466 // If exceptions are enabled, do not use parameter to avoid possible crash.
1467 if (auto *OMPRegionInfo =
1468 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1469 if (OMPRegionInfo->getThreadIDVariable()) {
1470 // Check if this an outlined function with thread id passed as argument.
1471 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1472 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1473 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1474 !CGF.getLangOpts().CXXExceptions ||
1475 CGF.Builder.GetInsertBlock() == TopBlock ||
1476 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1477 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1478 TopBlock ||
1479 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1480 CGF.Builder.GetInsertBlock()) {
1481 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1482 // If value loaded in entry block, cache it and use it everywhere in
1483 // function.
1484 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1485 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1486 Elem.second.ThreadID = ThreadID;
1487 }
1488 return ThreadID;
1489 }
1490 }
1491 }
1492
1493 // This is not an outlined function region - need to call __kmpc_int32
1494 // kmpc_global_thread_num(ident_t *loc).
1495 // Generate thread id value and cache this value for use across the
1496 // function.
1497 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1498 if (!Elem.second.ServiceInsertPt)
1499 setLocThreadIdInsertPt(CGF);
1500 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1501 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1502 llvm::CallInst *Call = CGF.Builder.CreateCall(
1503 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1504 OMPRTL___kmpc_global_thread_num),
1505 emitUpdateLocation(CGF, Loc));
1506 Call->setCallingConv(CGF.getRuntimeCC());
1507 Elem.second.ThreadID = Call;
1508 return Call;
1509}
1510
1511void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1512 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1512, __PRETTY_FUNCTION__))
;
1513 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1514 clearLocThreadIdInsertPt(CGF);
1515 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1516 }
1517 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1518 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1519 UDRMap.erase(D);
1520 FunctionUDRMap.erase(CGF.CurFn);
1521 }
1522 auto I = FunctionUDMMap.find(CGF.CurFn);
1523 if (I != FunctionUDMMap.end()) {
1524 for(const auto *D : I->second)
1525 UDMMap.erase(D);
1526 FunctionUDMMap.erase(I);
1527 }
1528 LastprivateConditionalToTypes.erase(CGF.CurFn);
1529 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1530}
1531
1532llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1533 return OMPBuilder.IdentPtr;
1534}
1535
1536llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1537 if (!Kmpc_MicroTy) {
1538 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1539 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1540 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1541 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1542 }
1543 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1544}
1545
1546llvm::FunctionCallee
1547CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
1548 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
1549 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
;
1550 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
1551 : "__kmpc_for_static_init_4u")
1552 : (IVSigned ? "__kmpc_for_static_init_8"
1553 : "__kmpc_for_static_init_8u");
1554 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1555 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1556 llvm::Type *TypeParams[] = {
1557 getIdentTyPointerTy(), // loc
1558 CGM.Int32Ty, // tid
1559 CGM.Int32Ty, // schedtype
1560 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1561 PtrTy, // p_lower
1562 PtrTy, // p_upper
1563 PtrTy, // p_stride
1564 ITy, // incr
1565 ITy // chunk
1566 };
1567 auto *FnTy =
1568 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1569 return CGM.CreateRuntimeFunction(FnTy, Name);
1570}
1571
1572llvm::FunctionCallee
1573CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
1574 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
1575 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
;
1576 StringRef Name =
1577 IVSize == 32
1578 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
1579 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
1580 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1581 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
1582 CGM.Int32Ty, // tid
1583 CGM.Int32Ty, // schedtype
1584 ITy, // lower
1585 ITy, // upper
1586 ITy, // stride
1587 ITy // chunk
1588 };
1589 auto *FnTy =
1590 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1591 return CGM.CreateRuntimeFunction(FnTy, Name);
1592}
1593
1594llvm::FunctionCallee
1595CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
1596 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
1597 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
;
1598 StringRef Name =
1599 IVSize == 32
1600 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
1601 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
1602 llvm::Type *TypeParams[] = {
1603 getIdentTyPointerTy(), // loc
1604 CGM.Int32Ty, // tid
1605 };
1606 auto *FnTy =
1607 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1608 return CGM.CreateRuntimeFunction(FnTy, Name);
1609}
1610
1611llvm::FunctionCallee
1612CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
1613 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
1614 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
;
1615 StringRef Name =
1616 IVSize == 32
1617 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
1618 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
1619 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1620 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1621 llvm::Type *TypeParams[] = {
1622 getIdentTyPointerTy(), // loc
1623 CGM.Int32Ty, // tid
1624 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1625 PtrTy, // p_lower
1626 PtrTy, // p_upper
1627 PtrTy // p_stride
1628 };
1629 auto *FnTy =
1630 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1631 return CGM.CreateRuntimeFunction(FnTy, Name);
1632}
1633
1634/// Obtain information that uniquely identifies a target entry. This
1635/// consists of the file and device IDs as well as line number associated with
1636/// the relevant entry source location.
1637static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
1638 unsigned &DeviceID, unsigned &FileID,
1639 unsigned &LineNum) {
1640 SourceManager &SM = C.getSourceManager();
1641
1642 // The loc should be always valid and have a file ID (the user cannot use
1643 // #pragma directives in macros)
1644
1645 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1645, __PRETTY_FUNCTION__))
;
1646
1647 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
1648 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1648, __PRETTY_FUNCTION__))
;
1649
1650 llvm::sys::fs::UniqueID ID;
1651 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
1652 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1653 << PLoc.getFilename() << EC.message();
1654
1655 DeviceID = ID.getDevice();
1656 FileID = ID.getFile();
1657 LineNum = PLoc.getLine();
1658}
1659
1660Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1661 if (CGM.getLangOpts().OpenMPSimd)
1662 return Address::invalid();
1663 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1664 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1665 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1666 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1667 HasRequiresUnifiedSharedMemory))) {
1668 SmallString<64> PtrName;
1669 {
1670 llvm::raw_svector_ostream OS(PtrName);
1671 OS << CGM.getMangledName(GlobalDecl(VD));
1672 if (!VD->isExternallyVisible()) {
1673 unsigned DeviceID, FileID, Line;
1674 getTargetEntryUniqueInfo(CGM.getContext(),
1675 VD->getCanonicalDecl()->getBeginLoc(),
1676 DeviceID, FileID, Line);
1677 OS << llvm::format("_%x", FileID);
1678 }
1679 OS << "_decl_tgt_ref_ptr";
1680 }
1681 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
1682 if (!Ptr) {
1683 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
1684 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
1685 PtrName);
1686
1687 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1688 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1689
1690 if (!CGM.getLangOpts().OpenMPIsDevice)
1691 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
1692 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
1693 }
1694 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
1695 }
1696 return Address::invalid();
1697}
1698
1699llvm::Constant *
1700CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1701 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
1702 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
;
1703 // Lookup the entry, lazily creating it if necessary.
1704 std::string Suffix = getName({"cache", ""});
1705 return getOrCreateInternalVariable(
1706 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
1707}
1708
1709Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1710 const VarDecl *VD,
1711 Address VDAddr,
1712 SourceLocation Loc) {
1713 if (CGM.getLangOpts().OpenMPUseTLS &&
1714 CGM.getContext().getTargetInfo().isTLSSupported())
1715 return VDAddr;
1716
1717 llvm::Type *VarTy = VDAddr.getElementType();
1718 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1719 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1720 CGM.Int8PtrTy),
1721 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1722 getOrCreateThreadPrivateCache(VD)};
1723 return Address(CGF.EmitRuntimeCall(
1724 OMPBuilder.getOrCreateRuntimeFunction(
1725 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1726 Args),
1727 VDAddr.getAlignment());
1728}
1729
1730void CGOpenMPRuntime::emitThreadPrivateVarInit(
1731 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1732 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1733 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1734 // library.
1735 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1736 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1737 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1738 OMPLoc);
1739 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1740 // to register constructor/destructor for variable.
1741 llvm::Value *Args[] = {
1742 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
1743 Ctor, CopyCtor, Dtor};
1744 CGF.EmitRuntimeCall(
1745 OMPBuilder.getOrCreateRuntimeFunction(
1746 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1747 Args);
1748}
1749
1750llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1751 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1752 bool PerformInit, CodeGenFunction *CGF) {
1753 if (CGM.getLangOpts().OpenMPUseTLS &&
1754 CGM.getContext().getTargetInfo().isTLSSupported())
1755 return nullptr;
1756
1757 VD = VD->getDefinition(CGM.getContext());
1758 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
1759 QualType ASTTy = VD->getType();
1760
1761 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1762 const Expr *Init = VD->getAnyInitializer();
1763 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1764 // Generate function that re-emits the declaration's initializer into the
1765 // threadprivate copy of the variable VD
1766 CodeGenFunction CtorCGF(CGM);
1767 FunctionArgList Args;
1768 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1769 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1770 ImplicitParamDecl::Other);
1771 Args.push_back(&Dst);
1772
1773 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1774 CGM.getContext().VoidPtrTy, Args);
1775 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1776 std::string Name = getName({"__kmpc_global_ctor_", ""});
1777 llvm::Function *Fn =
1778 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1779 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1780 Args, Loc, Loc);
1781 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1782 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1783 CGM.getContext().VoidPtrTy, Dst.getLocation());
1784 Address Arg = Address(ArgVal, VDAddr.getAlignment());
1785 Arg = CtorCGF.Builder.CreateElementBitCast(
1786 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
1787 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1788 /*IsInitializer=*/true);
1789 ArgVal = CtorCGF.EmitLoadOfScalar(
1790 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1791 CGM.getContext().VoidPtrTy, Dst.getLocation());
1792 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1793 CtorCGF.FinishFunction();
1794 Ctor = Fn;
1795 }
1796 if (VD->getType().isDestructedType() != QualType::DK_none) {
1797 // Generate function that emits destructor call for the threadprivate copy
1798 // of the variable VD
1799 CodeGenFunction DtorCGF(CGM);
1800 FunctionArgList Args;
1801 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1802 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1803 ImplicitParamDecl::Other);
1804 Args.push_back(&Dst);
1805
1806 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1807 CGM.getContext().VoidTy, Args);
1808 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1809 std::string Name = getName({"__kmpc_global_dtor_", ""});
1810 llvm::Function *Fn =
1811 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1812 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1813 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1814 Loc, Loc);
1815 // Create a scope with an artificial location for the body of this function.
1816 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1817 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1818 DtorCGF.GetAddrOfLocalVar(&Dst),
1819 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
1820 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
1821 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1822 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1823 DtorCGF.FinishFunction();
1824 Dtor = Fn;
1825 }
1826 // Do not emit init function if it is not required.
1827 if (!Ctor && !Dtor)
1828 return nullptr;
1829
1830 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1831 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1832 /*isVarArg=*/false)
1833 ->getPointerTo();
1834 // Copying constructor for the threadprivate variable.
1835 // Must be NULL - reserved by runtime, but currently it requires that this
1836 // parameter is always NULL. Otherwise it fires assertion.
1837 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1838 if (Ctor == nullptr) {
1839 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1840 /*isVarArg=*/false)
1841 ->getPointerTo();
1842 Ctor = llvm::Constant::getNullValue(CtorTy);
1843 }
1844 if (Dtor == nullptr) {
1845 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1846 /*isVarArg=*/false)
1847 ->getPointerTo();
1848 Dtor = llvm::Constant::getNullValue(DtorTy);
1849 }
1850 if (!CGF) {
1851 auto *InitFunctionTy =
1852 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1853 std::string Name = getName({"__omp_threadprivate_init_", ""});
1854 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1855 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
1856 CodeGenFunction InitCGF(CGM);
1857 FunctionArgList ArgList;
1858 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1859 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1860 Loc, Loc);
1861 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1862 InitCGF.FinishFunction();
1863 return InitFunction;
1864 }
1865 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1866 }
1867 return nullptr;
1868}
1869
1870bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
1871 llvm::GlobalVariable *Addr,
1872 bool PerformInit) {
1873 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
1874 !CGM.getLangOpts().OpenMPIsDevice)
1875 return false;
1876 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1877 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1878 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1879 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1880 HasRequiresUnifiedSharedMemory))
1881 return CGM.getLangOpts().OpenMPIsDevice;
1882 VD = VD->getDefinition(CGM.getContext());
1883 assert(VD && "Unknown VarDecl")((VD && "Unknown VarDecl") ? static_cast<void> (
0) : __assert_fail ("VD && \"Unknown VarDecl\"", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1883, __PRETTY_FUNCTION__))
;
1884
1885 if (!DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
1886 return CGM.getLangOpts().OpenMPIsDevice;
1887
1888 QualType ASTTy = VD->getType();
1889 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
1890
1891 // Produce the unique prefix to identify the new target regions. We use
1892 // the source location of the variable declaration which we know to not
1893 // conflict with any target region.
1894 unsigned DeviceID;
1895 unsigned FileID;
1896 unsigned Line;
1897 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
1898 SmallString<128> Buffer, Out;
1899 {
1900 llvm::raw_svector_ostream OS(Buffer);
1901 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
1902 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
1903 }
1904
1905 const Expr *Init = VD->getAnyInitializer();
1906 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1907 llvm::Constant *Ctor;
1908 llvm::Constant *ID;
1909 if (CGM.getLangOpts().OpenMPIsDevice) {
1910 // Generate function that re-emits the declaration's initializer into
1911 // the threadprivate copy of the variable VD
1912 CodeGenFunction CtorCGF(CGM);
1913
1914 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1915 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1916 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1917 FTy, Twine(Buffer, "_ctor"), FI, Loc);
1918 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
1919 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1920 FunctionArgList(), Loc, Loc);
1921 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
1922 CtorCGF.EmitAnyExprToMem(Init,
1923 Address(Addr, CGM.getContext().getDeclAlign(VD)),
1924 Init->getType().getQualifiers(),
1925 /*IsInitializer=*/true);
1926 CtorCGF.FinishFunction();
1927 Ctor = Fn;
1928 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1929 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
1930 } else {
1931 Ctor = new llvm::GlobalVariable(
1932 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1933 llvm::GlobalValue::PrivateLinkage,
1934 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
1935 ID = Ctor;
1936 }
1937
1938 // Register the information for the entry associated with the constructor.
1939 Out.clear();
1940 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1941 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
1942 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
1943 }
1944 if (VD->getType().isDestructedType() != QualType::DK_none) {
1945 llvm::Constant *Dtor;
1946 llvm::Constant *ID;
1947 if (CGM.getLangOpts().OpenMPIsDevice) {
1948 // Generate function that emits destructor call for the threadprivate
1949 // copy of the variable VD
1950 CodeGenFunction DtorCGF(CGM);
1951
1952 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1953 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1954 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1955 FTy, Twine(Buffer, "_dtor"), FI, Loc);
1956 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1957 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1958 FunctionArgList(), Loc, Loc);
1959 // Create a scope with an artificial location for the body of this
1960 // function.
1961 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1962 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
1963 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1964 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1965 DtorCGF.FinishFunction();
1966 Dtor = Fn;
1967 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1968 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
1969 } else {
1970 Dtor = new llvm::GlobalVariable(
1971 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1972 llvm::GlobalValue::PrivateLinkage,
1973 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
1974 ID = Dtor;
1975 }
1976 // Register the information for the entry associated with the destructor.
1977 Out.clear();
1978 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1979 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
1980 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
1981 }
1982 return CGM.getLangOpts().OpenMPIsDevice;
1983}
1984
1985Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1986 QualType VarType,
1987 StringRef Name) {
1988 std::string Suffix = getName({"artificial", ""});
1989 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1990 llvm::Value *GAddr =
1991 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
1992 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1993 CGM.getTarget().isTLSSupported()) {
1994 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
1995 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
1996 }
1997 std::string CacheSuffix = getName({"cache", ""});
1998 llvm::Value *Args[] = {
1999 emitUpdateLocation(CGF, SourceLocation()),
2000 getThreadID(CGF, SourceLocation()),
2001 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2002 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2003 /*isSigned=*/false),
2004 getOrCreateInternalVariable(
2005 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2006 return Address(
2007 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2008 CGF.EmitRuntimeCall(
2009 OMPBuilder.getOrCreateRuntimeFunction(
2010 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
2011 Args),
2012 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2013 CGM.getContext().getTypeAlignInChars(VarType));
2014}
2015
2016void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
2017 const RegionCodeGenTy &ThenGen,
2018 const RegionCodeGenTy &ElseGen) {
2019 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2020
2021 // If the condition constant folds and can be elided, try to avoid emitting
2022 // the condition and the dead arm of the if/else.
2023 bool CondConstant;
2024 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2025 if (CondConstant)
2026 ThenGen(CGF);
2027 else
2028 ElseGen(CGF);
2029 return;
2030 }
2031
2032 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2033 // emit the conditional branch.
2034 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2035 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2036 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2037 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2038
2039 // Emit the 'then' code.
2040 CGF.EmitBlock(ThenBlock);
2041 ThenGen(CGF);
2042 CGF.EmitBranch(ContBlock);
2043 // Emit the 'else' code if present.
2044 // There is no need to emit line number for unconditional branch.
2045 (void)ApplyDebugLocation::CreateEmpty(CGF);
2046 CGF.EmitBlock(ElseBlock);
2047 ElseGen(CGF);
2048 // There is no need to emit line number for unconditional branch.
2049 (void)ApplyDebugLocation::CreateEmpty(CGF);
2050 CGF.EmitBranch(ContBlock);
2051 // Emit the continuation block for code after the if.
2052 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2053}
2054
2055void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2056 llvm::Function *OutlinedFn,
2057 ArrayRef<llvm::Value *> CapturedVars,
2058 const Expr *IfCond) {
2059 if (!CGF.HaveInsertPoint())
2060 return;
2061 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2062 auto &M = CGM.getModule();
2063 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2064 this](CodeGenFunction &CGF, PrePostActionTy &) {
2065 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2066 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2067 llvm::Value *Args[] = {
2068 RTLoc,
2069 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2070 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2071 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2072 RealArgs.append(std::begin(Args), std::end(Args));
2073 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2074
2075 llvm::FunctionCallee RTLFn =
2076 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2077 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2078 };
2079 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2080 this](CodeGenFunction &CGF, PrePostActionTy &) {
2081 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2082 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2083 // Build calls:
2084 // __kmpc_serialized_parallel(&Loc, GTid);
2085 llvm::Value *Args[] = {RTLoc, ThreadID};
2086 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2087 M, OMPRTL___kmpc_serialized_parallel),
2088 Args);
2089
2090 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2091 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2092 Address ZeroAddrBound =
2093 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2094 /*Name=*/".bound.zero.addr");
2095 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
2096 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2097 // ThreadId for serialized parallels is 0.
2098 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2099 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
2100 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2101 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2102
2103 // __kmpc_end_serialized_parallel(&Loc, GTid);
2104 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2105 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2106 M, OMPRTL___kmpc_end_serialized_parallel),
2107 EndArgs);
2108 };
2109 if (IfCond) {
2110 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2111 } else {
2112 RegionCodeGenTy ThenRCG(ThenGen);
2113 ThenRCG(CGF);
2114 }
2115}
2116
2117// If we're inside an (outlined) parallel region, use the region info's
2118// thread-ID variable (it is passed in a first argument of the outlined function
2119// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2120// regular serial code region, get thread ID by calling kmp_int32
2121// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2122// return the address of that temp.
2123Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2124 SourceLocation Loc) {
2125 if (auto *OMPRegionInfo =
2126 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2127 if (OMPRegionInfo->getThreadIDVariable())
2128 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2129
2130 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2131 QualType Int32Ty =
2132 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2133 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2134 CGF.EmitStoreOfScalar(ThreadID,
2135 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2136
2137 return ThreadIDTemp;
2138}
2139
2140llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
2141 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2142 SmallString<256> Buffer;
2143 llvm::raw_svector_ostream Out(Buffer);
2144 Out << Name;
2145 StringRef RuntimeName = Out.str();
2146 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2147 if (Elem.second) {
2148 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2149, __PRETTY_FUNCTION__))
2149 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2149, __PRETTY_FUNCTION__))
;
2150 return &*Elem.second;
2151 }
2152
2153 return Elem.second = new llvm::GlobalVariable(
2154 CGM.getModule(), Ty, /*IsConstant*/ false,
2155 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2156 Elem.first(), /*InsertBefore=*/nullptr,
2157 llvm::GlobalValue::NotThreadLocal, AddressSpace);
2158}
2159
2160llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2161 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2162 std::string Name = getName({Prefix, "var"});
2163 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2164}
2165
2166namespace {
2167/// Common pre(post)-action for different OpenMP constructs.
2168class CommonActionTy final : public PrePostActionTy {
2169 llvm::FunctionCallee EnterCallee;
2170 ArrayRef<llvm::Value *> EnterArgs;
2171 llvm::FunctionCallee ExitCallee;
2172 ArrayRef<llvm::Value *> ExitArgs;
2173 bool Conditional;
2174 llvm::BasicBlock *ContBlock = nullptr;
2175
2176public:
2177 CommonActionTy(llvm::FunctionCallee EnterCallee,
2178 ArrayRef<llvm::Value *> EnterArgs,
2179 llvm::FunctionCallee ExitCallee,
2180 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2181 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2182 ExitArgs(ExitArgs), Conditional(Conditional) {}
2183 void Enter(CodeGenFunction &CGF) override {
2184 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2185 if (Conditional) {
2186 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2187 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2188 ContBlock = CGF.createBasicBlock("omp_if.end");
2189 // Generate the branch (If-stmt)
2190 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2191 CGF.EmitBlock(ThenBlock);
2192 }
2193 }
2194 void Done(CodeGenFunction &CGF) {
2195 // Emit the rest of blocks/branches
2196 CGF.EmitBranch(ContBlock);
2197 CGF.EmitBlock(ContBlock, true);
2198 }
2199 void Exit(CodeGenFunction &CGF) override {
2200 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2201 }
2202};
2203} // anonymous namespace
2204
2205void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2206 StringRef CriticalName,
2207 const RegionCodeGenTy &CriticalOpGen,
2208 SourceLocation Loc, const Expr *Hint) {
2209 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2210 // CriticalOpGen();
2211 // __kmpc_end_critical(ident_t *, gtid, Lock);
2212 // Prepare arguments and build a call to __kmpc_critical
2213 if (!CGF.HaveInsertPoint())
2214 return;
2215 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2216 getCriticalRegionLock(CriticalName)};
2217 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2218 std::end(Args));
2219 if (Hint) {
2220 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2221 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2222 }
2223 CommonActionTy Action(
2224 OMPBuilder.getOrCreateRuntimeFunction(
2225 CGM.getModule(),
2226 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2227 EnterArgs,
2228 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2229 OMPRTL___kmpc_end_critical),
2230 Args);
2231 CriticalOpGen.setAction(Action);
2232 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2233}
2234
2235void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2236 const RegionCodeGenTy &MasterOpGen,
2237 SourceLocation Loc) {
2238 if (!CGF.HaveInsertPoint())
2239 return;
2240 // if(__kmpc_master(ident_t *, gtid)) {
2241 // MasterOpGen();
2242 // __kmpc_end_master(ident_t *, gtid);
2243 // }
2244 // Prepare arguments and build a call to __kmpc_master
2245 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2246 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2247 CGM.getModule(), OMPRTL___kmpc_master),
2248 Args,
2249 OMPBuilder.getOrCreateRuntimeFunction(
2250 CGM.getModule(), OMPRTL___kmpc_end_master),
2251 Args,
2252 /*Conditional=*/true);
2253 MasterOpGen.setAction(Action);
2254 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2255 Action.Done(CGF);
2256}
2257
2258void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2259 SourceLocation Loc) {
2260 if (!CGF.HaveInsertPoint())
2261 return;
2262 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2263 OMPBuilder.createTaskyield(CGF.Builder);
2264 } else {
2265 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2266 llvm::Value *Args[] = {
2267 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2268 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
2269 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2270 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2271 Args);
2272 }
2273
2274 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2275 Region->emitUntiedSwitch(CGF);
2276}
2277
2278void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2279 const RegionCodeGenTy &TaskgroupOpGen,
2280 SourceLocation Loc) {
2281 if (!CGF.HaveInsertPoint())
2282 return;
2283 // __kmpc_taskgroup(ident_t *, gtid);
2284 // TaskgroupOpGen();
2285 // __kmpc_end_taskgroup(ident_t *, gtid);
2286 // Prepare arguments and build a call to __kmpc_taskgroup
2287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2288 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2289 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2290 Args,
2291 OMPBuilder.getOrCreateRuntimeFunction(
2292 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2293 Args);
2294 TaskgroupOpGen.setAction(Action);
2295 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2296}
2297
2298/// Given an array of pointers to variables, project the address of a
2299/// given variable.
2300static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2301 unsigned Index, const VarDecl *Var) {
2302 // Pull out the pointer to the variable.
2303 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2304 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2305
2306 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
2307 Addr = CGF.Builder.CreateElementBitCast(
2308 Addr, CGF.ConvertTypeForMem(Var->getType()));
2309 return Addr;
2310}
2311
2312static llvm::Value *emitCopyprivateCopyFunction(
2313 CodeGenModule &CGM, llvm::Type *ArgsType,
2314 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2315 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2316 SourceLocation Loc) {
2317 ASTContext &C = CGM.getContext();
2318 // void copy_func(void *LHSArg, void *RHSArg);
2319 FunctionArgList Args;
2320 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2321 ImplicitParamDecl::Other);
2322 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2323 ImplicitParamDecl::Other);
2324 Args.push_back(&LHSArg);
2325 Args.push_back(&RHSArg);
2326 const auto &CGFI =
2327 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
2328 std::string Name =
2329 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
2330 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
2331 llvm::GlobalValue::InternalLinkage, Name,
2332 &CGM.getModule());
2333 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
2334 Fn->setDoesNotRecurse();
2335 CodeGenFunction CGF(CGM);
2336 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2337 // Dest = (void*[n])(LHSArg);
2338 // Src = (void*[n])(RHSArg);
2339 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2340 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2341 ArgsType), CGF.getPointerAlign());
2342 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2343 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2344 ArgsType), CGF.getPointerAlign());
2345 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2346 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2347 // ...
2348 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2349 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2350 const auto *DestVar =
2351 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2352 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2353
2354 const auto *SrcVar =
2355 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2356 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2357
2358 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2359 QualType Type = VD->getType();
2360 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2361 }
2362 CGF.FinishFunction();
2363 return Fn;
2364}
2365
2366void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2367 const RegionCodeGenTy &SingleOpGen,
2368 SourceLocation Loc,
2369 ArrayRef<const Expr *> CopyprivateVars,
2370 ArrayRef<const Expr *> SrcExprs,
2371 ArrayRef<const Expr *> DstExprs,
2372 ArrayRef<const Expr *> AssignmentOps) {
2373 if (!CGF.HaveInsertPoint())
2374 return;
2375 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
2376 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
2377 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2377, __PRETTY_FUNCTION__))
;
2378 ASTContext &C = CGM.getContext();
2379 // int32 did_it = 0;
2380 // if(__kmpc_single(ident_t *, gtid)) {
2381 // SingleOpGen();
2382 // __kmpc_end_single(ident_t *, gtid);
2383 // did_it = 1;
2384 // }
2385 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2386 // <copy_func>, did_it);
2387
2388 Address DidIt = Address::invalid();
2389 if (!CopyprivateVars.empty()) {
2390 // int32 did_it = 0;
2391 QualType KmpInt32Ty =
2392 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2393 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2394 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2395 }
2396 // Prepare arguments and build a call to __kmpc_single
2397 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2398 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2399 CGM.getModule(), OMPRTL___kmpc_single),
2400 Args,
2401 OMPBuilder.getOrCreateRuntimeFunction(
2402 CGM.getModule(), OMPRTL___kmpc_end_single),
2403 Args,
2404 /*Conditional=*/true);
2405 SingleOpGen.setAction(Action);
2406 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2407 if (DidIt.isValid()) {
2408 // did_it = 1;
2409 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2410 }
2411 Action.Done(CGF);
2412 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2413 // <copy_func>, did_it);
2414 if (DidIt.isValid()) {
2415 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2416 QualType CopyprivateArrayTy = C.getConstantArrayType(
2417 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
2418 /*IndexTypeQuals=*/0);
2419 // Create a list of all private variables for copyprivate.
2420 Address CopyprivateList =
2421 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2422 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2423 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2424 CGF.Builder.CreateStore(
2425 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2426 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2427 CGF.VoidPtrTy),
2428 Elem);
2429 }
2430 // Build function that copies private values from single region to all other
2431 // threads in the corresponding parallel region.
2432 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2433 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
2434 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
2435 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2436 Address CL =
2437 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2438 CGF.VoidPtrTy);
2439 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2440 llvm::Value *Args[] = {
2441 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2442 getThreadID(CGF, Loc), // i32 <gtid>
2443 BufSize, // size_t <buf_size>
2444 CL.getPointer(), // void *<copyprivate list>
2445 CpyFn, // void (*) (void *, void *) <copy_func>
2446 DidItVal // i32 did_it
2447 };
2448 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2449 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2450 Args);
2451 }
2452}
2453
2454void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2455 const RegionCodeGenTy &OrderedOpGen,
2456 SourceLocation Loc, bool IsThreads) {
2457 if (!CGF.HaveInsertPoint())
2458 return;
2459 // __kmpc_ordered(ident_t *, gtid);
2460 // OrderedOpGen();
2461 // __kmpc_end_ordered(ident_t *, gtid);
2462 // Prepare arguments and build a call to __kmpc_ordered
2463 if (IsThreads) {
2464 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2465 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2466 CGM.getModule(), OMPRTL___kmpc_ordered),
2467 Args,
2468 OMPBuilder.getOrCreateRuntimeFunction(
2469 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2470 Args);
2471 OrderedOpGen.setAction(Action);
2472 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2473 return;
2474 }
2475 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2476}
2477
2478unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2479 unsigned Flags;
2480 if (Kind == OMPD_for)
2481 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2482 else if (Kind == OMPD_sections)
2483 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2484 else if (Kind == OMPD_single)
2485 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2486 else if (Kind == OMPD_barrier)
2487 Flags = OMP_IDENT_BARRIER_EXPL;
2488 else
2489 Flags = OMP_IDENT_BARRIER_IMPL;
2490 return Flags;
2491}
2492
2493void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2494 CodeGenFunction &CGF, const OMPLoopDirective &S,
2495 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2496 // Check if the loop directive is actually a doacross loop directive. In this
2497 // case choose static, 1 schedule.
2498 if (llvm::any_of(
2499 S.getClausesOfKind<OMPOrderedClause>(),
2500 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2501 ScheduleKind = OMPC_SCHEDULE_static;
2502 // Chunk size is 1 in this case.
2503 llvm::APInt ChunkSize(32, 1);
2504 ChunkExpr = IntegerLiteral::Create(
2505 CGF.getContext(), ChunkSize,
2506 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2507 SourceLocation());
2508 }
2509}
2510
2511void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2512 OpenMPDirectiveKind Kind, bool EmitChecks,
2513 bool ForceSimpleCall) {
2514 // Check if we should use the OMPBuilder
2515 auto *OMPRegionInfo =
2516 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2517 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2518 CGF.Builder.restoreIP(OMPBuilder.createBarrier(
2519 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
2520 return;
2521 }
2522
2523 if (!CGF.HaveInsertPoint())
2524 return;
2525 // Build call __kmpc_cancel_barrier(loc, thread_id);
2526 // Build call __kmpc_barrier(loc, thread_id);
2527 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2528 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2529 // thread_id);
2530 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2531 getThreadID(CGF, Loc)};
2532 if (OMPRegionInfo) {
2533 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2534 llvm::Value *Result = CGF.EmitRuntimeCall(
2535 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2536 OMPRTL___kmpc_cancel_barrier),
2537 Args);
2538 if (EmitChecks) {
2539 // if (__kmpc_cancel_barrier()) {
2540 // exit from construct;
2541 // }
2542 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2543 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2544 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2545 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2546 CGF.EmitBlock(ExitBB);
2547 // exit from construct;
2548 CodeGenFunction::JumpDest CancelDestination =
2549 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2550 CGF.EmitBranchThroughCleanup(CancelDestination);
2551 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2552 }
2553 return;
2554 }
2555 }
2556 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2557 CGM.getModule(), OMPRTL___kmpc_barrier),
2558 Args);
2559}
2560
2561/// Map the OpenMP loop schedule to the runtime enumeration.
2562static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2563 bool Chunked, bool Ordered) {
2564 switch (ScheduleKind) {
2565 case OMPC_SCHEDULE_static:
2566 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2567 : (Ordered ? OMP_ord_static : OMP_sch_static);
2568 case OMPC_SCHEDULE_dynamic:
2569 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2570 case OMPC_SCHEDULE_guided:
2571 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2572 case OMPC_SCHEDULE_runtime:
2573 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2574 case OMPC_SCHEDULE_auto:
2575 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2576 case OMPC_SCHEDULE_unknown:
2577 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2577, __PRETTY_FUNCTION__))
;
2578 return Ordered ? OMP_ord_static : OMP_sch_static;
2579 }
2580 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2580)
;
2581}
2582
2583/// Map the OpenMP distribute schedule to the runtime enumeration.
2584static OpenMPSchedType
2585getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2586 // only static is allowed for dist_schedule
2587 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2588}
2589
2590bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2591 bool Chunked) const {
2592 OpenMPSchedType Schedule =
2593 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2594 return Schedule == OMP_sch_static;
2595}
2596
2597bool CGOpenMPRuntime::isStaticNonchunked(
2598 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2599 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2600 return Schedule == OMP_dist_sch_static;
2601}
2602
2603bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2604 bool Chunked) const {
2605 OpenMPSchedType Schedule =
2606 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2607 return Schedule == OMP_sch_static_chunked;
2608}
2609
2610bool CGOpenMPRuntime::isStaticChunked(
2611 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2612 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2613 return Schedule == OMP_dist_sch_static_chunked;
2614}
2615
2616bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2617 OpenMPSchedType Schedule =
2618 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2619 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2619, __PRETTY_FUNCTION__))
;
2620 return Schedule != OMP_sch_static;
2621}
2622
2623static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2624 OpenMPScheduleClauseModifier M1,
2625 OpenMPScheduleClauseModifier M2) {
2626 int Modifier = 0;
2627 switch (M1) {
2628 case OMPC_SCHEDULE_MODIFIER_monotonic:
2629 Modifier = OMP_sch_modifier_monotonic;
2630 break;
2631 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2632 Modifier = OMP_sch_modifier_nonmonotonic;
2633 break;
2634 case OMPC_SCHEDULE_MODIFIER_simd:
2635 if (Schedule == OMP_sch_static_chunked)
2636 Schedule = OMP_sch_static_balanced_chunked;
2637 break;
2638 case OMPC_SCHEDULE_MODIFIER_last:
2639 case OMPC_SCHEDULE_MODIFIER_unknown:
2640 break;
2641 }
2642 switch (M2) {
2643 case OMPC_SCHEDULE_MODIFIER_monotonic:
2644 Modifier = OMP_sch_modifier_monotonic;
2645 break;
2646 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2647 Modifier = OMP_sch_modifier_nonmonotonic;
2648 break;
2649 case OMPC_SCHEDULE_MODIFIER_simd:
2650 if (Schedule == OMP_sch_static_chunked)
2651 Schedule = OMP_sch_static_balanced_chunked;
2652 break;
2653 case OMPC_SCHEDULE_MODIFIER_last:
2654 case OMPC_SCHEDULE_MODIFIER_unknown:
2655 break;
2656 }
2657 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2658 // If the static schedule kind is specified or if the ordered clause is
2659 // specified, and if the nonmonotonic modifier is not specified, the effect is
2660 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2661 // modifier is specified, the effect is as if the nonmonotonic modifier is
2662 // specified.
2663 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2664 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2665 Schedule == OMP_sch_static_balanced_chunked ||
2666 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2667 Schedule == OMP_dist_sch_static_chunked ||
2668 Schedule == OMP_dist_sch_static))
2669 Modifier = OMP_sch_modifier_nonmonotonic;
2670 }
2671 return Schedule | Modifier;
2672}
2673
2674void CGOpenMPRuntime::emitForDispatchInit(
2675 CodeGenFunction &CGF, SourceLocation Loc,
2676 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2677 bool Ordered, const DispatchRTInput &DispatchValues) {
2678 if (!CGF.HaveInsertPoint())
2679 return;
2680 OpenMPSchedType Schedule = getRuntimeSchedule(
2681 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
2682 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2683 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2684 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
2685 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2685, __PRETTY_FUNCTION__))
;
2686 // Call __kmpc_dispatch_init(
2687 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2688 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2689 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2690
2691 // If the Chunk was not specified in the clause - use default value 1.
2692 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2693 : CGF.Builder.getIntN(IVSize, 1);
2694 llvm::Value *Args[] = {
2695 emitUpdateLocation(CGF, Loc),
2696 getThreadID(CGF, Loc),
2697 CGF.Builder.getInt32(addMonoNonMonoModifier(
2698 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
2699 DispatchValues.LB, // Lower
2700 DispatchValues.UB, // Upper
2701 CGF.Builder.getIntN(IVSize, 1), // Stride
2702 Chunk // Chunk
2703 };
2704 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2705}
2706
2707static void emitForStaticInitCall(
2708 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2709 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2710 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2711 const CGOpenMPRuntime::StaticRTInput &Values) {
2712 if (!CGF.HaveInsertPoint())
2713 return;
2714
2715 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2715, __PRETTY_FUNCTION__))
;
2716 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2717 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2718 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2719 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
2720 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2720, __PRETTY_FUNCTION__))
;
2721
2722 // Call __kmpc_for_static_init(
2723 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2724 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2725 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2726 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2727 llvm::Value *Chunk = Values.Chunk;
2728 if (Chunk == nullptr) {
2729 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
2730 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
2731 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2731, __PRETTY_FUNCTION__))
;
2732 // If the Chunk was not specified in the clause - use default value 1.
2733 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2734 } else {
2735 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2736 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2737 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2738 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2739 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
;
2740 }
2741 llvm::Value *Args[] = {
2742 UpdateLocation,
2743 ThreadId,
2744 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2745 M2)), // Schedule type
2746 Values.IL.getPointer(), // &isLastIter
2747 Values.LB.getPointer(), // &LB
2748 Values.UB.getPointer(), // &UB
2749 Values.ST.getPointer(), // &Stride
2750 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2751 Chunk // Chunk
2752 };
2753 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2754}
2755
2756void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2757 SourceLocation Loc,
2758 OpenMPDirectiveKind DKind,
2759 const OpenMPScheduleTy &ScheduleKind,
2760 const StaticRTInput &Values) {
2761 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
2762 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
2763 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2764, __PRETTY_FUNCTION__))
2764 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2764, __PRETTY_FUNCTION__))
;
2765 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2766 isOpenMPLoopDirective(DKind)
2767 ? OMP_IDENT_WORK_LOOP
2768 : OMP_IDENT_WORK_SECTIONS);
2769 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2770 llvm::FunctionCallee StaticInitFunction =
2771 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2772 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2773 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2774 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
2775}
2776
2777void CGOpenMPRuntime::emitDistributeStaticInit(
2778 CodeGenFunction &CGF, SourceLocation Loc,
2779 OpenMPDistScheduleClauseKind SchedKind,
2780 const CGOpenMPRuntime::StaticRTInput &Values) {
2781 OpenMPSchedType ScheduleNum =
2782 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
2783 llvm::Value *UpdatedLocation =
2784 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2785 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2786 llvm::FunctionCallee StaticInitFunction =
2787 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2788 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2789 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
2790 OMPC_SCHEDULE_MODIFIER_unknown, Values);
2791}
2792
2793void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2794 SourceLocation Loc,
2795 OpenMPDirectiveKind DKind) {
2796 if (!CGF.HaveInsertPoint())
2797 return;
2798 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2799 llvm::Value *Args[] = {
2800 emitUpdateLocation(CGF, Loc,
2801 isOpenMPDistributeDirective(DKind)
2802 ? OMP_IDENT_WORK_DISTRIBUTE
2803 : isOpenMPLoopDirective(DKind)
2804 ? OMP_IDENT_WORK_LOOP
2805 : OMP_IDENT_WORK_SECTIONS),
2806 getThreadID(CGF, Loc)};
2807 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2808 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2809 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2810 Args);
2811}
2812
2813void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2814 SourceLocation Loc,
2815 unsigned IVSize,
2816 bool IVSigned) {
2817 if (!CGF.HaveInsertPoint())
2818 return;
2819 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2820 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2821 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2822}
2823
2824llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2825 SourceLocation Loc, unsigned IVSize,
2826 bool IVSigned, Address IL,
2827 Address LB, Address UB,
2828 Address ST) {
2829 // Call __kmpc_dispatch_next(
2830 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2831 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2832 // kmp_int[32|64] *p_stride);
2833 llvm::Value *Args[] = {
2834 emitUpdateLocation(CGF, Loc),
2835 getThreadID(CGF, Loc),
2836 IL.getPointer(), // &isLastIter
2837 LB.getPointer(), // &Lower
2838 UB.getPointer(), // &Upper
2839 ST.getPointer() // &Stride
2840 };
2841 llvm::Value *Call =
2842 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2843 return CGF.EmitScalarConversion(
2844 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2845 CGF.getContext().BoolTy, Loc);
2846}
2847
2848void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2849 llvm::Value *NumThreads,
2850 SourceLocation Loc) {
2851 if (!CGF.HaveInsertPoint())
2852 return;
2853 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2854 llvm::Value *Args[] = {
2855 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2856 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2857 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2858 CGM.getModule(), OMPRTL___kmpc_push_num_threads),
2859 Args);
2860}
2861
2862void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2863 ProcBindKind ProcBind,
2864 SourceLocation Loc) {
2865 if (!CGF.HaveInsertPoint())
2866 return;
2867 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2867, __PRETTY_FUNCTION__))
;
2868 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2869 llvm::Value *Args[] = {
2870 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2871 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
2872 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2873 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2874 Args);
2875}
2876
2877void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2878 SourceLocation Loc, llvm::AtomicOrdering AO) {
2879 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2880 OMPBuilder.createFlush(CGF.Builder);
2881 } else {
2882 if (!CGF.HaveInsertPoint())
2883 return;
2884 // Build call void __kmpc_flush(ident_t *loc)
2885 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2886 CGM.getModule(), OMPRTL___kmpc_flush),
2887 emitUpdateLocation(CGF, Loc));
2888 }
2889}
2890
2891namespace {
2892/// Indexes of fields for type kmp_task_t.
2893enum KmpTaskTFields {
2894 /// List of shared variables.
2895 KmpTaskTShareds,
2896 /// Task routine.
2897 KmpTaskTRoutine,
2898 /// Partition id for the untied tasks.
2899 KmpTaskTPartId,
2900 /// Function with call of destructors for private variables.
2901 Data1,
2902 /// Task priority.
2903 Data2,
2904 /// (Taskloops only) Lower bound.
2905 KmpTaskTLowerBound,
2906 /// (Taskloops only) Upper bound.
2907 KmpTaskTUpperBound,
2908 /// (Taskloops only) Stride.
2909 KmpTaskTStride,
2910 /// (Taskloops only) Is last iteration flag.
2911 KmpTaskTLastIter,
2912 /// (Taskloops only) Reduction data.
2913 KmpTaskTReductions,
2914};
2915} // anonymous namespace
2916
2917bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
2918 return OffloadEntriesTargetRegion.empty() &&
2919 OffloadEntriesDeviceGlobalVar.empty();
2920}
2921
2922/// Initialize target region entry.
2923void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2924 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2925 StringRef ParentName, unsigned LineNum,
2926 unsigned Order) {
2927 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
2928 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
2929 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2929, __PRETTY_FUNCTION__))
;
2930 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
2931 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
2932 OMPTargetRegionEntryTargetRegion);
2933 ++OffloadingEntriesNum;
2934}
2935
2936void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2937 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2938 StringRef ParentName, unsigned LineNum,
2939 llvm::Constant *Addr, llvm::Constant *ID,
2940 OMPTargetRegionEntryKind Flags) {
2941 // If we are emitting code for a target, the entry is already initialized,
2942 // only has to be registered.
2943 if (CGM.getLangOpts().OpenMPIsDevice) {
2944 // This could happen if the device compilation is invoked standalone.
2945 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum))
2946 initializeTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2947 OffloadingEntriesNum);
2948 auto &Entry =
2949 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
2950 Entry.setAddress(Addr);
2951 Entry.setID(ID);
2952 Entry.setFlags(Flags);
2953 } else {
2954 if (Flags ==
2955 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion &&
2956 hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2957 /*IgnoreAddressId*/ true))
2958 return;
2959 assert(!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2960, __PRETTY_FUNCTION__))
2960 "Target region entry already registered!")((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2960, __PRETTY_FUNCTION__))
;
2961 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
2962 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
2963 ++OffloadingEntriesNum;
2964 }
2965}
2966
2967bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2968 unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum,
2969 bool IgnoreAddressId) const {
2970 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2971 if (PerDevice == OffloadEntriesTargetRegion.end())
2972 return false;
2973 auto PerFile = PerDevice->second.find(FileID);
2974 if (PerFile == PerDevice->second.end())
2975 return false;
2976 auto PerParentName = PerFile->second.find(ParentName);
2977 if (PerParentName == PerFile->second.end())
2978 return false;
2979 auto PerLine = PerParentName->second.find(LineNum);
2980 if (PerLine == PerParentName->second.end())
2981 return false;
2982 // Fail if this entry is already registered.
2983 if (!IgnoreAddressId &&
2984 (PerLine->second.getAddress() || PerLine->second.getID()))
2985 return false;
2986 return true;
2987}
2988
2989void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
2990 const OffloadTargetRegionEntryInfoActTy &Action) {
2991 // Scan all target region entries and perform the provided action.
2992 for (const auto &D : OffloadEntriesTargetRegion)
2993 for (const auto &F : D.second)
2994 for (const auto &P : F.second)
2995 for (const auto &L : P.second)
2996 Action(D.first, F.first, P.first(), L.first, L.second);
2997}
2998
2999void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3000 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3001 OMPTargetGlobalVarEntryKind Flags,
3002 unsigned Order) {
3003 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3005, __PRETTY_FUNCTION__))
3004 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3005, __PRETTY_FUNCTION__))
3005 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3005, __PRETTY_FUNCTION__))
;
3006 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3007 ++OffloadingEntriesNum;
3008}
3009
3010void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3011 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3012 CharUnits VarSize,
3013 OMPTargetGlobalVarEntryKind Flags,
3014 llvm::GlobalValue::LinkageTypes Linkage) {
3015 if (CGM.getLangOpts().OpenMPIsDevice) {
3016 // This could happen if the device compilation is invoked standalone.
3017 if (!hasDeviceGlobalVarEntryInfo(VarName))
3018 initializeDeviceGlobalVarEntryInfo(VarName, Flags, OffloadingEntriesNum);
3019 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3020 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3021, __PRETTY_FUNCTION__))
3021 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3021, __PRETTY_FUNCTION__))
;
3022 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3023 if (Entry.getVarSize().isZero()) {
3024 Entry.setVarSize(VarSize);
3025 Entry.setLinkage(Linkage);
3026 }
3027 return;
3028 }
3029 Entry.setVarSize(VarSize);
3030 Entry.setLinkage(Linkage);
3031 Entry.setAddress(Addr);
3032 } else {
3033 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3034 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3035 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3036, __PRETTY_FUNCTION__))
3036 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3036, __PRETTY_FUNCTION__))
;
3037 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3038, __PRETTY_FUNCTION__))
3038 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3038, __PRETTY_FUNCTION__))
;
3039 if (Entry.getVarSize().isZero()) {
3040 Entry.setVarSize(VarSize);
3041 Entry.setLinkage(Linkage);
3042 }
3043 return;
3044 }
3045 OffloadEntriesDeviceGlobalVar.try_emplace(
3046 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3047 ++OffloadingEntriesNum;
3048 }
3049}
3050
3051void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3052 actOnDeviceGlobalVarEntriesInfo(
3053 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3054 // Scan all target region entries and perform the provided action.
3055 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3056 Action(E.getKey(), E.getValue());
3057}
3058
3059void CGOpenMPRuntime::createOffloadEntry(
3060 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3061 llvm::GlobalValue::LinkageTypes Linkage) {
3062 StringRef Name = Addr->getName();
3063 llvm::Module &M = CGM.getModule();
3064 llvm::LLVMContext &C = M.getContext();
3065
3066 // Create constant string with the name.
3067 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3068
3069 std::string StringName = getName({"omp_offloading", "entry_name"});
3070 auto *Str = new llvm::GlobalVariable(
3071 M, StrPtrInit->getType(), /*isConstant=*/true,
3072 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3073 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3074
3075 llvm::Constant *Data[] = {
3076 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(ID, CGM.VoidPtrTy),
3077 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, CGM.Int8PtrTy),
3078 llvm::ConstantInt::get(CGM.SizeTy, Size),
3079 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3080 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3081 std::string EntryName = getName({"omp_offloading", "entry", ""});
3082 llvm::GlobalVariable *Entry = createGlobalStruct(
3083 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
3084 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
3085
3086 // The entry has to be created in the section the linker expects it to be.
3087 Entry->setSection("omp_offloading_entries");
3088}
3089
3090void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3091 // Emit the offloading entries and metadata so that the device codegen side
3092 // can easily figure out what to emit. The produced metadata looks like
3093 // this:
3094 //
3095 // !omp_offload.info = !{!1, ...}
3096 //
3097 // Right now we only generate metadata for function that contain target
3098 // regions.
3099
3100 // If we are in simd mode or there are no entries, we don't need to do
3101 // anything.
3102 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
3103 return;
3104
3105 llvm::Module &M = CGM.getModule();
3106 llvm::LLVMContext &C = M.getContext();
3107 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
3108 SourceLocation, StringRef>,
3109 16>
3110 OrderedEntries(OffloadEntriesInfoManager.size());
3111 llvm::SmallVector<StringRef, 16> ParentFunctions(
3112 OffloadEntriesInfoManager.size());
3113
3114 // Auxiliary methods to create metadata values and strings.
3115 auto &&GetMDInt = [this](unsigned V) {
3116 return llvm::ConstantAsMetadata::get(
3117 llvm::ConstantInt::get(CGM.Int32Ty, V));
3118 };
3119
3120 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3121
3122 // Create the offloading info metadata node.
3123 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3124
3125 // Create function that emits metadata for each target region entry;
3126 auto &&TargetRegionMetadataEmitter =
3127 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
3128 &GetMDString](
3129 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3130 unsigned Line,
3131 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3132 // Generate metadata for target regions. Each entry of this metadata
3133 // contains:
3134 // - Entry 0 -> Kind of this type of metadata (0).
3135 // - Entry 1 -> Device ID of the file where the entry was identified.
3136 // - Entry 2 -> File ID of the file where the entry was identified.
3137 // - Entry 3 -> Mangled name of the function where the entry was
3138 // identified.
3139 // - Entry 4 -> Line in the file where the entry was identified.
3140 // - Entry 5 -> Order the entry was created.
3141 // The first element of the metadata node is the kind.
3142 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3143 GetMDInt(FileID), GetMDString(ParentName),
3144 GetMDInt(Line), GetMDInt(E.getOrder())};
3145
3146 SourceLocation Loc;
3147 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3148 E = CGM.getContext().getSourceManager().fileinfo_end();
3149 I != E; ++I) {
3150 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
3151 I->getFirst()->getUniqueID().getFile() == FileID) {
3152 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3153 I->getFirst(), Line, 1);
3154 break;
3155 }
3156 }
3157 // Save this entry in the right position of the ordered entries array.
3158 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
3159 ParentFunctions[E.getOrder()] = ParentName;
3160
3161 // Add metadata to the named metadata node.
3162 MD->addOperand(llvm::MDNode::get(C, Ops));
3163 };
3164
3165 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3166 TargetRegionMetadataEmitter);
3167
3168 // Create function that emits metadata for each device global variable entry;
3169 auto &&DeviceGlobalVarMetadataEmitter =
3170 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3171 MD](StringRef MangledName,
3172 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3173 &E) {
3174 // Generate metadata for global variables. Each entry of this metadata
3175 // contains:
3176 // - Entry 0 -> Kind of this type of metadata (1).
3177 // - Entry 1 -> Mangled name of the variable.
3178 // - Entry 2 -> Declare target kind.
3179 // - Entry 3 -> Order the entry was created.
3180 // The first element of the metadata node is the kind.
3181 llvm::Metadata *Ops[] = {
3182 GetMDInt(E.getKind()), GetMDString(MangledName),
3183 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3184
3185 // Save this entry in the right position of the ordered entries array.
3186 OrderedEntries[E.getOrder()] =
3187 std::make_tuple(&E, SourceLocation(), MangledName);
3188
3189 // Add metadata to the named metadata node.
3190 MD->addOperand(llvm::MDNode::get(C, Ops));
3191 };
3192
3193 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3194 DeviceGlobalVarMetadataEmitter);
3195
3196 for (const auto &E : OrderedEntries) {
3197 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3197, __PRETTY_FUNCTION__))
;
3198 if (const auto *CE =
3199 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3200 std::get<0>(E))) {
3201 if (!CE->getID() || !CE->getAddress()) {
3202 // Do not blame the entry if the parent funtion is not emitted.
3203 StringRef FnName = ParentFunctions[CE->getOrder()];
3204 if (!CGM.GetGlobalValue(FnName))
3205 continue;
3206 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3207 DiagnosticsEngine::Error,
3208 "Offloading entry for target region in %0 is incorrect: either the "
3209 "address or the ID is invalid.");
3210 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
3211 continue;
3212 }
3213 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
3214 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3215 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
3216 OffloadEntryInfoDeviceGlobalVar>(
3217 std::get<0>(E))) {
3218 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3219 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3220 CE->getFlags());
3221 switch (Flags) {
3222 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3223 if (CGM.getLangOpts().OpenMPIsDevice &&
3224 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
3225 continue;
3226 if (!CE->getAddress()) {
3227 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3228 DiagnosticsEngine::Error, "Offloading entry for declare target "
3229 "variable %0 is incorrect: the "
3230 "address is invalid.");
3231 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
3232 continue;
3233 }
3234 // The vaiable has no definition - no need to add the entry.
3235 if (CE->getVarSize().isZero())
3236 continue;
3237 break;
3238 }
3239 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3240 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3242, __PRETTY_FUNCTION__))
3241 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3242, __PRETTY_FUNCTION__))
3242 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3242, __PRETTY_FUNCTION__))
;
3243 if (CGM.getLangOpts().OpenMPIsDevice)
3244 continue;
3245 if (!CE->getAddress()) {
3246 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3247 DiagnosticsEngine::Error,
3248 "Offloading entry for declare target variable is incorrect: the "
3249 "address is invalid.");
3250 CGM.getDiags().Report(DiagID);
3251 continue;
3252 }
3253 break;
3254 }
3255 createOffloadEntry(CE->getAddress(), CE->getAddress(),
3256 CE->getVarSize().getQuantity(), Flags,
3257 CE->getLinkage());
3258 } else {
3259 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3259)
;
3260 }
3261 }
3262}
3263
3264/// Loads all the offload entries information from the host IR
3265/// metadata.
3266void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3267 // If we are in target mode, load the metadata from the host IR. This code has
3268 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
3269
3270 if (!CGM.getLangOpts().OpenMPIsDevice)
3271 return;
3272
3273 if (CGM.getLangOpts().OMPHostIRFile.empty())
3274 return;
3275
3276 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3277 if (auto EC = Buf.getError()) {
3278 CGM.getDiags().Report(diag::err_cannot_open_file)
3279 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3280 return;
3281 }
3282
3283 llvm::LLVMContext C;
3284 auto ME = expectedToErrorOrAndEmitErrors(
3285 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3286
3287 if (auto EC = ME.getError()) {
3288 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3289 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
3290 CGM.getDiags().Report(DiagID)
3291 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3292 return;
3293 }
3294
3295 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
3296 if (!MD)
3297 return;
3298
3299 for (llvm::MDNode *MN : MD->operands()) {
3300 auto &&GetMDInt = [MN](unsigned Idx) {
3301 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3302 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3303 };
3304
3305 auto &&GetMDString = [MN](unsigned Idx) {
3306 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
3307 return V->getString();
3308 };
3309
3310 switch (GetMDInt(0)) {
3311 default:
3312 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3312)
;
3313 break;
3314 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3315 OffloadingEntryInfoTargetRegion:
3316 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3317 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
3318 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
3319 /*Order=*/GetMDInt(5));
3320 break;
3321 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3322 OffloadingEntryInfoDeviceGlobalVar:
3323 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
3324 /*MangledName=*/GetMDString(1),
3325 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3326 /*Flags=*/GetMDInt(2)),
3327 /*Order=*/GetMDInt(3));
3328 break;
3329 }
3330 }
3331}
3332
3333void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3334 if (!KmpRoutineEntryPtrTy) {
3335 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3336 ASTContext &C = CGM.getContext();
3337 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3338 FunctionProtoType::ExtProtoInfo EPI;
3339 KmpRoutineEntryPtrQTy = C.getPointerType(
3340 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3341 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3342 }
3343}
3344
3345QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
3346 // Make sure the type of the entry is already created. This is the type we
3347 // have to create:
3348 // struct __tgt_offload_entry{
3349 // void *addr; // Pointer to the offload entry info.
3350 // // (function or global)
3351 // char *name; // Name of the function or global.
3352 // size_t size; // Size of the entry info (0 if it a function).
3353 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
3354 // int32_t reserved; // Reserved, to use by the runtime library.
3355 // };
3356 if (TgtOffloadEntryQTy.isNull()) {
3357 ASTContext &C = CGM.getContext();
3358 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
3359 RD->startDefinition();
3360 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3361 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
3362 addFieldToRecordDecl(C, RD, C.getSizeType());
3363 addFieldToRecordDecl(
3364 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3365 addFieldToRecordDecl(
3366 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3367 RD->completeDefinition();
3368 RD->addAttr(PackedAttr::CreateImplicit(C));
3369 TgtOffloadEntryQTy = C.getRecordType(RD);
3370 }
3371 return TgtOffloadEntryQTy;
3372}
3373
3374namespace {
3375struct PrivateHelpersTy {
3376 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3377 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3378 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3379 PrivateElemInit(PrivateElemInit) {}
3380 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3381 const Expr *OriginalRef = nullptr;
3382 const VarDecl *Original = nullptr;
3383 const VarDecl *PrivateCopy = nullptr;
3384 const VarDecl *PrivateElemInit = nullptr;
3385 bool isLocalPrivate() const {
3386 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3387 }
3388};
3389typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3390} // anonymous namespace
3391
3392static bool isAllocatableDecl(const VarDecl *VD) {
3393 const VarDecl *CVD = VD->getCanonicalDecl();
3394 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3395 return false;
3396 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3397 // Use the default allocation.
3398 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
3399 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
3400 !AA->getAllocator());
3401}
3402
3403static RecordDecl *
3404createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3405 if (!Privates.empty()) {
3406 ASTContext &C = CGM.getContext();
3407 // Build struct .kmp_privates_t. {
3408 // /* private vars */
3409 // };
3410 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
3411 RD->startDefinition();
3412 for (const auto &Pair : Privates) {
3413 const VarDecl *VD = Pair.second.Original;
3414 QualType Type = VD->getType().getNonReferenceType();
3415 // If the private variable is a local variable with lvalue ref type,
3416 // allocate the pointer instead of the pointee type.
3417 if (Pair.second.isLocalPrivate()) {
3418 if (VD->getType()->isLValueReferenceType())
3419 Type = C.getPointerType(Type);
3420 if (isAllocatableDecl(VD))
3421 Type = C.getPointerType(Type);
3422 }
3423 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
3424 if (VD->hasAttrs()) {
3425 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3426 E(VD->getAttrs().end());
3427 I != E; ++I)
3428 FD->addAttr(*I);
3429 }
3430 }
3431 RD->completeDefinition();
3432 return RD;
3433 }
3434 return nullptr;
3435}
3436
3437static RecordDecl *
3438createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3439 QualType KmpInt32Ty,
3440 QualType KmpRoutineEntryPointerQTy) {
3441 ASTContext &C = CGM.getContext();
3442 // Build struct kmp_task_t {
3443 // void * shareds;
3444 // kmp_routine_entry_t routine;
3445 // kmp_int32 part_id;
3446 // kmp_cmplrdata_t data1;
3447 // kmp_cmplrdata_t data2;
3448 // For taskloops additional fields:
3449 // kmp_uint64 lb;
3450 // kmp_uint64 ub;
3451 // kmp_int64 st;
3452 // kmp_int32 liter;
3453 // void * reductions;
3454 // };
3455 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
3456 UD->startDefinition();
3457 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3458 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3459 UD->completeDefinition();
3460 QualType KmpCmplrdataTy = C.getRecordType(UD);
3461 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
3462 RD->startDefinition();
3463 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3464 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3465 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3466 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3467 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3468 if (isOpenMPTaskLoopDirective(Kind)) {
3469 QualType KmpUInt64Ty =
3470 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3471 QualType KmpInt64Ty =
3472 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3473 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3474 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3475 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3476 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3477 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3478 }
3479 RD->completeDefinition();
3480 return RD;
3481}
3482
3483static RecordDecl *
3484createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3485 ArrayRef<PrivateDataTy> Privates) {
3486 ASTContext &C = CGM.getContext();
3487 // Build struct kmp_task_t_with_privates {
3488 // kmp_task_t task_data;
3489 // .kmp_privates_t. privates;
3490 // };
3491 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3492 RD->startDefinition();
3493 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
3494 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3495 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
3496 RD->completeDefinition();
3497 return RD;
3498}
3499
3500/// Emit a proxy function which accepts kmp_task_t as the second
3501/// argument.
3502/// \code
3503/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3504/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3505/// For taskloops:
3506/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3507/// tt->reductions, tt->shareds);
3508/// return 0;
3509/// }
3510/// \endcode
3511static llvm::Function *
3512emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3513 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3514 QualType KmpTaskTWithPrivatesPtrQTy,
3515 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3516 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3517 llvm::Value *TaskPrivatesMap) {
3518 ASTContext &C = CGM.getContext();
3519 FunctionArgList Args;
3520 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3521 ImplicitParamDecl::Other);
3522 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3523 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3524 ImplicitParamDecl::Other);
3525 Args.push_back(&GtidArg);
3526 Args.push_back(&TaskTypeArg);
3527 const auto &TaskEntryFnInfo =
3528 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3529 llvm::FunctionType *TaskEntryTy =
3530 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3531 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
3532 auto *TaskEntry = llvm::Function::Create(
3533 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3534 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
3535 TaskEntry->setDoesNotRecurse();
3536 CodeGenFunction CGF(CGM);
3537 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3538 Loc, Loc);
3539
3540 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3541 // tt,
3542 // For taskloops:
3543 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3544 // tt->task_data.shareds);
3545 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3546 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3547 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3548 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3549 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3550 const auto *KmpTaskTWithPrivatesQTyRD =
3551 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3552 LValue Base =
3553 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3554 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
3555 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3556 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3557 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3558
3559 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3560 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3561 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3562 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3563 CGF.ConvertTypeForMem(SharedsPtrTy));
3564
3565 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3566 llvm::Value *PrivatesParam;
3567 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3568 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3569 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3570 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3571 } else {
3572 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3573 }
3574
3575 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3576 TaskPrivatesMap,
3577 CGF.Builder
3578 .CreatePointerBitCastOrAddrSpaceCast(
3579 TDBase.getAddress(CGF), CGF.VoidPtrTy)
3580 .getPointer()};
3581 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3582 std::end(CommonArgs));
3583 if (isOpenMPTaskLoopDirective(Kind)) {
3584 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3585 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3586 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3587 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3588 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3589 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3590 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3591 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3592 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3593 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3594 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3595 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3596 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3597 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3598 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3599 CallArgs.push_back(LBParam);
3600 CallArgs.push_back(UBParam);
3601 CallArgs.push_back(StParam);
3602 CallArgs.push_back(LIParam);
3603 CallArgs.push_back(RParam);
3604 }
3605 CallArgs.push_back(SharedsParam);
3606
3607 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3608 CallArgs);
3609 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3610 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3611 CGF.FinishFunction();
3612 return TaskEntry;
3613}
3614
3615static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3616 SourceLocation Loc,
3617 QualType KmpInt32Ty,
3618 QualType KmpTaskTWithPrivatesPtrQTy,
3619 QualType KmpTaskTWithPrivatesQTy) {
3620 ASTContext &C = CGM.getContext();
3621 FunctionArgList Args;
3622 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3623 ImplicitParamDecl::Other);
3624 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3625 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3626 ImplicitParamDecl::Other);
3627 Args.push_back(&GtidArg);
3628 Args.push_back(&TaskTypeArg);
3629 const auto &DestructorFnInfo =
3630 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3631 llvm::FunctionType *DestructorFnTy =
3632 CGM.getTypes().GetFunctionType(DestructorFnInfo);
3633 std::string Name =
3634 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
3635 auto *DestructorFn =
3636 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3637 Name, &CGM.getModule());
3638 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
3639 DestructorFnInfo);
3640 DestructorFn->setDoesNotRecurse();
3641 CodeGenFunction CGF(CGM);
3642 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3643 Args, Loc, Loc);
3644
3645 LValue Base = CGF.EmitLoadOfPointerLValue(
3646 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3647 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3648 const auto *KmpTaskTWithPrivatesQTyRD =
3649 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3650 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3651 Base = CGF.EmitLValueForField(Base, *FI);
3652 for (const auto *Field :
3653 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3654 if (QualType::DestructionKind DtorKind =
3655 Field->getType().isDestructedType()) {
3656 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3657 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
3658 }
3659 }
3660 CGF.FinishFunction();
3661 return DestructorFn;
3662}
3663
3664/// Emit a privates mapping function for correct handling of private and
3665/// firstprivate variables.
3666/// \code
3667/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3668/// **noalias priv1,..., <tyn> **noalias privn) {
3669/// *priv1 = &.privates.priv1;
3670/// ...;
3671/// *privn = &.privates.privn;
3672/// }
3673/// \endcode
3674static llvm::Value *
3675emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3676 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3677 ArrayRef<PrivateDataTy> Privates) {
3678 ASTContext &C = CGM.getContext();
3679 FunctionArgList Args;
3680 ImplicitParamDecl TaskPrivatesArg(
3681 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3682 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3683 ImplicitParamDecl::Other);
3684 Args.push_back(&TaskPrivatesArg);
3685 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3686 unsigned Counter = 1;
3687 for (const Expr *E : Data.PrivateVars) {
3688 Args.push_back(ImplicitParamDecl::Create(
3689 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3690 C.getPointerType(C.getPointerType(E->getType()))
3691 .withConst()
3692 .withRestrict(),
3693 ImplicitParamDecl::Other));
3694 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3695 PrivateVarsPos[VD] = Counter;
3696 ++Counter;
3697 }
3698 for (const Expr *E : Data.FirstprivateVars) {
3699 Args.push_back(ImplicitParamDecl::Create(
3700 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3701 C.getPointerType(C.getPointerType(E->getType()))
3702 .withConst()
3703 .withRestrict(),
3704 ImplicitParamDecl::Other));
3705 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3706 PrivateVarsPos[VD] = Counter;
3707 ++Counter;
3708 }
3709 for (const Expr *E : Data.LastprivateVars) {
3710 Args.push_back(ImplicitParamDecl::Create(
3711 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3712 C.getPointerType(C.getPointerType(E->getType()))
3713 .withConst()
3714 .withRestrict(),
3715 ImplicitParamDecl::Other));
3716 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3717 PrivateVarsPos[VD] = Counter;
3718 ++Counter;
3719 }
3720 for (const VarDecl *VD : Data.PrivateLocals) {
3721 QualType Ty = VD->getType().getNonReferenceType();
3722 if (VD->getType()->isLValueReferenceType())
3723 Ty = C.getPointerType(Ty);
3724 if (isAllocatableDecl(VD))
3725 Ty = C.getPointerType(Ty);
3726 Args.push_back(ImplicitParamDecl::Create(
3727 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3728 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3729 ImplicitParamDecl::Other));
3730 PrivateVarsPos[VD] = Counter;
3731 ++Counter;
3732 }
3733 const auto &TaskPrivatesMapFnInfo =
3734 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3735 llvm::FunctionType *TaskPrivatesMapTy =
3736 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
3737 std::string Name =
3738 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
3739 auto *TaskPrivatesMap = llvm::Function::Create(
3740 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3741 &CGM.getModule());
3742 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
3743 TaskPrivatesMapFnInfo);
3744 if (CGM.getLangOpts().Optimize) {
3745 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3746 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3747 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3748 }
3749 CodeGenFunction CGF(CGM);
3750 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3751 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3752
3753 // *privi = &.privates.privi;
3754 LValue Base = CGF.EmitLoadOfPointerLValue(
3755 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3756 TaskPrivatesArg.getType()->castAs<PointerType>());
3757 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
3758 Counter = 0;
3759 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3760 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3761 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3762 LValue RefLVal =
3763 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3764 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3765 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
3766 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3767 ++Counter;
3768 }
3769 CGF.FinishFunction();
3770 return TaskPrivatesMap;
3771}
3772
3773/// Emit initialization for private variables in task-based directives.
3774static void emitPrivatesInit(CodeGenFunction &CGF,
3775 const OMPExecutableDirective &D,
3776 Address KmpTaskSharedsPtr, LValue TDBase,
3777 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3778 QualType SharedsTy, QualType SharedsPtrTy,
3779 const OMPTaskDataTy &Data,
3780 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3781 ASTContext &C = CGF.getContext();
3782 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3783 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3784 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
3785 ? OMPD_taskloop
3786 : OMPD_task;
3787 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
3788 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3789 LValue SrcBase;
3790 bool IsTargetTask =
3791 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
3792 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
3793 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3794 // PointersArray, SizesArray, and MappersArray. The original variables for
3795 // these arrays are not captured and we get their addresses explicitly.
3796 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3797 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3798 SrcBase = CGF.MakeAddrLValue(
3799 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3800 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
3801 SharedsTy);
3802 }
3803 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3804 for (const PrivateDataTy &Pair : Privates) {
3805 // Do not initialize private locals.
3806 if (Pair.second.isLocalPrivate()) {
3807 ++FI;
3808 continue;
3809 }
3810 const VarDecl *VD = Pair.second.PrivateCopy;
3811 const Expr *Init = VD->getAnyInitializer();
3812 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3813 !CGF.isTrivialInitializer(Init)))) {
3814 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3815 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3816 const VarDecl *OriginalVD = Pair.second.Original;
3817 // Check if the variable is the target-based BasePointersArray,
3818 // PointersArray, SizesArray, or MappersArray.
3819 LValue SharedRefLValue;
3820 QualType Type = PrivateLValue.getType();
3821 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3822 if (IsTargetTask && !SharedField) {
3823 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3824 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3825 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3826 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3827 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3828 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3829 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
3830 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3830, __PRETTY_FUNCTION__))
;
3831 SharedRefLValue =
3832 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3833 } else if (ForDup) {
3834 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3835 SharedRefLValue = CGF.MakeAddrLValue(
3836 Address(SharedRefLValue.getPointer(CGF),
3837 C.getDeclAlign(OriginalVD)),
3838 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3839 SharedRefLValue.getTBAAInfo());
3840 } else if (CGF.LambdaCaptureFields.count(
3841 Pair.second.Original->getCanonicalDecl()) > 0 ||
3842 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) {
3843 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3844 } else {
3845 // Processing for implicitly captured variables.
3846 InlinedOpenMPRegionRAII Region(
3847 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3848 /*HasCancel=*/false);
3849 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3850 }
3851 if (Type->isArrayType()) {
3852 // Initialize firstprivate array.
3853 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3854 // Perform simple memcpy.
3855 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3856 } else {
3857 // Initialize firstprivate array using element-by-element
3858 // initialization.
3859 CGF.EmitOMPAggregateAssign(
3860 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
3861 Type,
3862 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3863 Address SrcElement) {
3864 // Clean up any temporaries needed by the initialization.
3865 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3866 InitScope.addPrivate(
3867 Elem, [SrcElement]() -> Address { return SrcElement; });
3868 (void)InitScope.Privatize();
3869 // Emit initialization for single element.
3870 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3871 CGF, &CapturesInfo);
3872 CGF.EmitAnyExprToMem(Init, DestElement,
3873 Init->getType().getQualifiers(),
3874 /*IsInitializer=*/false);
3875 });
3876 }
3877 } else {
3878 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3879 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
3880 return SharedRefLValue.getAddress(CGF);
3881 });
3882 (void)InitScope.Privatize();
3883 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3884 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3885 /*capturedByInit=*/false);
3886 }
3887 } else {
3888 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3889 }
3890 }
3891 ++FI;
3892 }
3893}
3894
3895/// Check if duplication function is required for taskloops.
3896static bool checkInitIsRequired(CodeGenFunction &CGF,
3897 ArrayRef<PrivateDataTy> Privates) {
3898 bool InitRequired = false;
3899 for (const PrivateDataTy &Pair : Privates) {
3900 if (Pair.second.isLocalPrivate())
3901 continue;
3902 const VarDecl *VD = Pair.second.PrivateCopy;
3903 const Expr *Init = VD->getAnyInitializer();
3904 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
3905 !CGF.isTrivialInitializer(Init));
3906 if (InitRequired)
3907 break;
3908 }
3909 return InitRequired;
3910}
3911
3912
3913/// Emit task_dup function (for initialization of
3914/// private/firstprivate/lastprivate vars and last_iter flag)
3915/// \code
3916/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3917/// lastpriv) {
3918/// // setup lastprivate flag
3919/// task_dst->last = lastpriv;
3920/// // could be constructor calls here...
3921/// }
3922/// \endcode
3923static llvm::Value *
3924emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3925 const OMPExecutableDirective &D,
3926 QualType KmpTaskTWithPrivatesPtrQTy,
3927 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3928 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3929 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3930 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3931 ASTContext &C = CGM.getContext();
3932 FunctionArgList Args;
3933 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3934 KmpTaskTWithPrivatesPtrQTy,
3935 ImplicitParamDecl::Other);
3936 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3937 KmpTaskTWithPrivatesPtrQTy,
3938 ImplicitParamDecl::Other);
3939 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
3940 ImplicitParamDecl::Other);
3941 Args.push_back(&DstArg);
3942 Args.push_back(&SrcArg);
3943 Args.push_back(&LastprivArg);
3944 const auto &TaskDupFnInfo =
3945 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3946 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
3947 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
3948 auto *TaskDup = llvm::Function::Create(
3949 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3950 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
3951 TaskDup->setDoesNotRecurse();
3952 CodeGenFunction CGF(CGM);
3953 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3954 Loc);
3955
3956 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3957 CGF.GetAddrOfLocalVar(&DstArg),
3958 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3959 // task_dst->liter = lastpriv;
3960 if (WithLastIter) {
3961 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3962 LValue Base = CGF.EmitLValueForField(
3963 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3964 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3965 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3966 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3967 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3968 }
3969
3970 // Emit initial values for private copies (if any).
3971 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3971, __PRETTY_FUNCTION__))
;
3972 Address KmpTaskSharedsPtr = Address::invalid();
3973 if (!Data.FirstprivateVars.empty()) {
3974 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3975 CGF.GetAddrOfLocalVar(&SrcArg),
3976 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3977 LValue Base = CGF.EmitLValueForField(
3978 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3979 KmpTaskSharedsPtr = Address(
3980 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3981 Base, *std::next(KmpTaskTQTyRD->field_begin(),
3982 KmpTaskTShareds)),
3983 Loc),
3984 CGM.getNaturalTypeAlignment(SharedsTy));
3985 }
3986 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3987 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3988 CGF.FinishFunction();
3989 return TaskDup;
3990}
3991
3992/// Checks if destructor function is required to be generated.
3993/// \return true if cleanups are required, false otherwise.
3994static bool
3995checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3996 ArrayRef<PrivateDataTy> Privates) {
3997 for (const PrivateDataTy &P : Privates) {
3998 if (P.second.isLocalPrivate())
3999 continue;
4000 QualType Ty = P.second.Original->getType().getNonReferenceType();
4001 if (Ty.isDestructedType())
4002 return true;
4003 }
4004 return false;
4005}
4006
4007namespace {
4008/// Loop generator for OpenMP iterator expression.
4009class OMPIteratorGeneratorScope final
4010 : public CodeGenFunction::OMPPrivateScope {
4011 CodeGenFunction &CGF;
4012 const OMPIteratorExpr *E = nullptr;
4013 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
4014 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
4015 OMPIteratorGeneratorScope() = delete;
4016 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
4017
4018public:
4019 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
4020 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
4021 if (!E)
4022 return;
4023 SmallVector<llvm::Value *, 4> Uppers;
4024 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4025 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
4026 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
4027 addPrivate(VD, [&CGF, VD]() {
4028 return CGF.CreateMemTemp(VD->getType(), VD->getName());
4029 });
4030 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4031 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() {
4032 return CGF.CreateMemTemp(HelperData.CounterVD->getType(),
4033 "counter.addr");
4034 });
4035 }
4036 Privatize();
4037
4038 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4039 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4040 LValue CLVal =
4041 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
4042 HelperData.CounterVD->getType());
4043 // Counter = 0;
4044 CGF.EmitStoreOfScalar(
4045 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0),
4046 CLVal);
4047 CodeGenFunction::JumpDest &ContDest =
4048 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
4049 CodeGenFunction::JumpDest &ExitDest =
4050 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
4051 // N = <number-of_iterations>;
4052 llvm::Value *N = Uppers[I];
4053 // cont:
4054 // if (Counter < N) goto body; else goto exit;
4055 CGF.EmitBlock(ContDest.getBlock());
4056 auto *CVal =
4057 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
4058 llvm::Value *Cmp =
4059 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
4060 ? CGF.Builder.CreateICmpSLT(CVal, N)
4061 : CGF.Builder.CreateICmpULT(CVal, N);
4062 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
4063 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
4064 // body:
4065 CGF.EmitBlock(BodyBB);
4066 // Iteri = Begini + Counter * Stepi;
4067 CGF.EmitIgnoredExpr(HelperData.Update);
4068 }
4069 }
4070 ~OMPIteratorGeneratorScope() {
4071 if (!E)
4072 return;
4073 for (unsigned I = E->numOfIterators(); I > 0; --I) {
4074 // Counter = Counter + 1;
4075 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);
4076 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
4077 // goto cont;
4078 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
4079 // exit:
4080 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
4081 }
4082 }
4083};
4084} // namespace
4085
4086static std::pair<llvm::Value *, llvm::Value *>
4087getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
4088 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
4089 llvm::Value *Addr;
4090 if (OASE) {
4091 const Expr *Base = OASE->getBase();
4092 Addr = CGF.EmitScalarExpr(Base);
4093 } else {
4094 Addr = CGF.EmitLValue(E).getPointer(CGF);
4095 }
4096 llvm::Value *SizeVal;
4097 QualType Ty = E->getType();
4098 if (OASE) {
4099 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
4100 for (const Expr *SE : OASE->getDimensions()) {
4101 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
4102 Sz = CGF.EmitScalarConversion(
4103 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
4104 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
4105 }
4106 } else if (const auto *ASE =
4107 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
4108 LValue UpAddrLVal =
4109 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
4110 llvm::Value *UpAddr =
4111 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
4112 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
4113 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
4114 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4115 } else {
4116 SizeVal = CGF.getTypeSize(Ty);
4117 }
4118 return std::make_pair(Addr, SizeVal);
4119}
4120
4121/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4122static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
4123 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);
4124 if (KmpTaskAffinityInfoTy.isNull()) {
4125 RecordDecl *KmpAffinityInfoRD =
4126 C.buildImplicitRecord("kmp_task_affinity_info_t");
4127 KmpAffinityInfoRD->startDefinition();
4128 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());
4129 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
4130 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
4131 KmpAffinityInfoRD->completeDefinition();
4132 KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
4133 }
4134}
4135
4136CGOpenMPRuntime::TaskResultTy
4137CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4138 const OMPExecutableDirective &D,
4139 llvm::Function *TaskFunction, QualType SharedsTy,
4140 Address Shareds, const OMPTaskDataTy &Data) {
4141 ASTContext &C = CGM.getContext();
4142 llvm::SmallVector<PrivateDataTy, 4> Privates;
4143 // Aggregate privates and sort them by the alignment.
4144 const auto *I = Data.PrivateCopies.begin();
4145 for (const Expr *E : Data.PrivateVars) {
4146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4147 Privates.emplace_back(
4148 C.getDeclAlign(VD),
4149 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4150 /*PrivateElemInit=*/nullptr));
4151 ++I;
4152 }
4153 I = Data.FirstprivateCopies.begin();
4154 const auto *IElemInitRef = Data.FirstprivateInits.begin();
4155 for (const Expr *E : Data.FirstprivateVars) {
4156 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4157 Privates.emplace_back(
4158 C.getDeclAlign(VD),
4159 PrivateHelpersTy(
4160 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4161 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4162 ++I;
4163 ++IElemInitRef;
4164 }
4165 I = Data.LastprivateCopies.begin();
4166 for (const Expr *E : Data.LastprivateVars) {
4167 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4168 Privates.emplace_back(
4169 C.getDeclAlign(VD),
4170 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4171 /*PrivateElemInit=*/nullptr));
4172 ++I;
4173 }
4174 for (const VarDecl *VD : Data.PrivateLocals) {
4175 if (isAllocatableDecl(VD))
4176 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));
4177 else
4178 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));
4179 }
4180 llvm::stable_sort(Privates,
4181 [](const PrivateDataTy &L, const PrivateDataTy &R) {
4182 return L.first > R.first;
4183 });
4184 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4185 // Build type kmp_routine_entry_t (if not built yet).
4186 emitKmpRoutineEntryT(KmpInt32Ty);
4187 // Build type kmp_task_t (if not built yet).
4188 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4189 if (SavedKmpTaskloopTQTy.isNull()) {
4190 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4191 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4192 }
4193 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4194 } else {
4195 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4198, __PRETTY_FUNCTION__))
4196 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4198, __PRETTY_FUNCTION__))
4197 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4198, __PRETTY_FUNCTION__))
4198 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4198, __PRETTY_FUNCTION__))
;
4199 if (SavedKmpTaskTQTy.isNull()) {
4200 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4201 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4202 }
4203 KmpTaskTQTy = SavedKmpTaskTQTy;
4204 }
4205 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4206 // Build particular struct kmp_task_t for the given task.
4207 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4208 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4209 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4210 QualType KmpTaskTWithPrivatesPtrQTy =
4211 C.getPointerType(KmpTaskTWithPrivatesQTy);
4212 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4213 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4214 KmpTaskTWithPrivatesTy->getPointerTo();
4215 llvm::Value *KmpTaskTWithPrivatesTySize =
4216 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4217 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4218
4219 // Emit initial values for private copies (if any).
4220 llvm::Value *TaskPrivatesMap = nullptr;
4221 llvm::Type *TaskPrivatesMapTy =
4222 std::next(TaskFunction->arg_begin(), 3)->getType();
4223 if (!Privates.empty()) {
4224 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4225 TaskPrivatesMap =
4226 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);
4227 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4228 TaskPrivatesMap, TaskPrivatesMapTy);
4229 } else {
4230 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4231 cast<llvm::PointerType>(TaskPrivatesMapTy));
4232 }
4233 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4234 // kmp_task_t *tt);
4235 llvm::Function *TaskEntry = emitProxyTaskFunction(
4236 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4237 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4238 TaskPrivatesMap);
4239
4240 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4241 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4242 // kmp_routine_entry_t *task_entry);
4243 // Task flags. Format is taken from
4244 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
4245 // description of kmp_tasking_flags struct.
4246 enum {
4247 TiedFlag = 0x1,
4248 FinalFlag = 0x2,
4249 DestructorsFlag = 0x8,
4250 PriorityFlag = 0x20,
4251 DetachableFlag = 0x40,
4252 };
4253 unsigned Flags = Data.Tied ? TiedFlag : 0;
4254 bool NeedsCleanup = false;
4255 if (!Privates.empty()) {
4256 NeedsCleanup =
4257 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4258 if (NeedsCleanup)
4259 Flags = Flags | DestructorsFlag;
4260 }
4261 if (Data.Priority.getInt())
4262 Flags = Flags | PriorityFlag;
4263 if (D.hasClausesOfKind<OMPDetachClause>())
4264 Flags = Flags | DetachableFlag;
4265 llvm::Value *TaskFlags =
4266 Data.Final.getPointer()
4267 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4268 CGF.Builder.getInt32(FinalFlag),
4269 CGF.Builder.getInt32(/*C=*/0))
4270 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4271 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4272 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4273 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4274 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4275 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4276 TaskEntry, KmpRoutineEntryPtrTy)};
4277 llvm::Value *NewTask;
4278 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4279 // Check if we have any device clause associated with the directive.
4280 const Expr *Device = nullptr;
4281 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4282 Device = C->getDevice();
4283 // Emit device ID if any otherwise use default value.
4284 llvm::Value *DeviceID;
4285 if (Device)
4286 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
4287 CGF.Int64Ty, /*isSigned=*/true);
4288 else
4289 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
4290 AllocArgs.push_back(DeviceID);
4291 NewTask = CGF.EmitRuntimeCall(
4292 OMPBuilder.getOrCreateRuntimeFunction(
4293 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4294 AllocArgs);
4295 } else {
4296 NewTask =
4297 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4298 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4299 AllocArgs);
4300 }
4301 // Emit detach clause initialization.
4302 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4303 // task_descriptor);
4304 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4305 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4306 LValue EvtLVal = CGF.EmitLValue(Evt);
4307
4308 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4309 // int gtid, kmp_task_t *task);
4310 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
4311 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
4312 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
4313 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4314 OMPBuilder.getOrCreateRuntimeFunction(
4315 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4316 {Loc, Tid, NewTask});
4317 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
4318 Evt->getExprLoc());
4319 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
4320 }
4321 // Process affinity clauses.
4322 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4323 // Process list of affinity data.
4324 ASTContext &C = CGM.getContext();
4325 Address AffinitiesArray = Address::invalid();
4326 // Calculate number of elements to form the array of affinity data.
4327 llvm::Value *NumOfElements = nullptr;
4328 unsigned NumAffinities = 0;
4329 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4330 if (const Expr *Modifier = C->getModifier()) {
4331 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4332 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4333 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4334 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4335 NumOfElements =
4336 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4337 }
4338 } else {
4339 NumAffinities += C->varlist_size();
4340 }
4341 }
4342 getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy);
4343 // Fields ids in kmp_task_affinity_info record.
4344 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4345
4346 QualType KmpTaskAffinityInfoArrayTy;
4347 if (NumOfElements) {
4348 NumOfElements = CGF.Builder.CreateNUWAdd(
4349 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
4350 OpaqueValueExpr OVE(
4351 Loc,
4352 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),
4353 VK_RValue);
4354 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4355 RValue::get(NumOfElements));
4356 KmpTaskAffinityInfoArrayTy =
4357 C.getVariableArrayType(KmpTaskAffinityInfoTy, &OVE, ArrayType::Normal,
4358 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4359 // Properly emit variable-sized array.
4360 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,
4361 ImplicitParamDecl::Other);
4362 CGF.EmitVarDecl(*PD);
4363 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
4364 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4365 /*isSigned=*/false);
4366 } else {
4367 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4368 KmpTaskAffinityInfoTy,
4369 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,
4370 ArrayType::Normal, /*IndexTypeQuals=*/0);
4371 AffinitiesArray =
4372 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
4373 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
4374 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,
4375 /*isSigned=*/false);
4376 }
4377
4378 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4379 // Fill array by elements without iterators.
4380 unsigned Pos = 0;
4381 bool HasIterator = false;
4382 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4383 if (C->getModifier()) {
4384 HasIterator = true;
4385 continue;
4386 }
4387 for (const Expr *E : C->varlists()) {
4388 llvm::Value *Addr;
4389 llvm::Value *Size;
4390 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4391 LValue Base =
4392 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
4393 KmpTaskAffinityInfoTy);
4394 // affs[i].base_addr = &<Affinities[i].second>;
4395 LValue BaseAddrLVal = CGF.EmitLValueForField(
4396 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4397 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4398 BaseAddrLVal);
4399 // affs[i].len = sizeof(<Affinities[i].second>);
4400 LValue LenLVal = CGF.EmitLValueForField(
4401 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4402 CGF.EmitStoreOfScalar(Size, LenLVal);
4403 ++Pos;
4404 }
4405 }
4406 LValue PosLVal;
4407 if (HasIterator) {
4408 PosLVal = CGF.MakeAddrLValue(
4409 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
4410 C.getSizeType());
4411 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4412 }
4413 // Process elements with iterators.
4414 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4415 const Expr *Modifier = C->getModifier();
4416 if (!Modifier)
4417 continue;
4418 OMPIteratorGeneratorScope IteratorScope(
4419 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
4420 for (const Expr *E : C->varlists()) {
4421 llvm::Value *Addr;
4422 llvm::Value *Size;
4423 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4424 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4425 LValue Base = CGF.MakeAddrLValue(
4426 Address(CGF.Builder.CreateGEP(AffinitiesArray.getPointer(), Idx),
4427 AffinitiesArray.getAlignment()),
4428 KmpTaskAffinityInfoTy);
4429 // affs[i].base_addr = &<Affinities[i].second>;
4430 LValue BaseAddrLVal = CGF.EmitLValueForField(
4431 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4432 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4433 BaseAddrLVal);
4434 // affs[i].len = sizeof(<Affinities[i].second>);
4435 LValue LenLVal = CGF.EmitLValueForField(
4436 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4437 CGF.EmitStoreOfScalar(Size, LenLVal);
4438 Idx = CGF.Builder.CreateNUWAdd(
4439 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4440 CGF.EmitStoreOfScalar(Idx, PosLVal);
4441 }
4442 }
4443 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4444 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4445 // naffins, kmp_task_affinity_info_t *affin_list);
4446 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4447 llvm::Value *GTid = getThreadID(CGF, Loc);
4448 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4449 AffinitiesArray.getPointer(), CGM.VoidPtrTy);
4450 // FIXME: Emit the function and ignore its result for now unless the
4451 // runtime function is properly implemented.
4452 (void)CGF.EmitRuntimeCall(
4453 OMPBuilder.getOrCreateRuntimeFunction(
4454 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4455 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4456 }
4457 llvm::Value *NewTaskNewTaskTTy =
4458 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4459 NewTask, KmpTaskTWithPrivatesPtrTy);
4460 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4461 KmpTaskTWithPrivatesQTy);
4462 LValue TDBase =
4463 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4464 // Fill the data in the resulting kmp_task_t record.
4465 // Copy shareds if there are any.
4466 Address KmpTaskSharedsPtr = Address::invalid();
4467 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4468 KmpTaskSharedsPtr =
4469 Address(CGF.EmitLoadOfScalar(
4470 CGF.EmitLValueForField(
4471 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4472 KmpTaskTShareds)),
4473 Loc),
4474 CGM.getNaturalTypeAlignment(SharedsTy));
4475 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4476 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4477 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4478 }
4479 // Emit initial values for private copies (if any).
4480 TaskResultTy Result;
4481 if (!Privates.empty()) {
4482 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4483 SharedsTy, SharedsPtrTy, Data, Privates,
4484 /*ForDup=*/false);
4485 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4486 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4487 Result.TaskDupFn = emitTaskDupFunction(
4488 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4489 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4490 /*WithLastIter=*/!Data.LastprivateVars.empty());
4491 }
4492 }
4493 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4494 enum { Priority = 0, Destructors = 1 };
4495 // Provide pointer to function with destructors for privates.
4496 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4497 const RecordDecl *KmpCmplrdataUD =
4498 (*FI)->getType()->getAsUnionType()->getDecl();
4499 if (NeedsCleanup) {
4500 llvm::Value *DestructorFn = emitDestructorsFunction(
4501 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4502 KmpTaskTWithPrivatesQTy);
4503 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4504 LValue DestructorsLV = CGF.EmitLValueForField(
4505 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4506 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4507 DestructorFn, KmpRoutineEntryPtrTy),
4508 DestructorsLV);
4509 }
4510 // Set priority.
4511 if (Data.Priority.getInt()) {
4512 LValue Data2LV = CGF.EmitLValueForField(
4513 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4514 LValue PriorityLV = CGF.EmitLValueForField(
4515 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4516 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4517 }
4518 Result.NewTask = NewTask;
4519 Result.TaskEntry = TaskEntry;
4520 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4521 Result.TDBase = TDBase;
4522 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4523 return Result;
4524}
4525
4526namespace {
4527/// Dependence kind for RTL.
4528enum RTLDependenceKindTy {
4529 DepIn = 0x01,
4530 DepInOut = 0x3,
4531 DepMutexInOutSet = 0x4
4532};
4533/// Fields ids in kmp_depend_info record.
4534enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4535} // namespace
4536
4537/// Translates internal dependency kind into the runtime kind.
4538static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4539 RTLDependenceKindTy DepKind;
4540 switch (K) {
4541 case OMPC_DEPEND_in:
4542 DepKind = DepIn;
4543 break;
4544 // Out and InOut dependencies must use the same code.
4545 case OMPC_DEPEND_out:
4546 case OMPC_DEPEND_inout:
4547 DepKind = DepInOut;
4548 break;
4549 case OMPC_DEPEND_mutexinoutset:
4550 DepKind = DepMutexInOutSet;
4551 break;
4552 case OMPC_DEPEND_source:
4553 case OMPC_DEPEND_sink:
4554 case OMPC_DEPEND_depobj:
4555 case OMPC_DEPEND_unknown:
4556 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4556)
;
4557 }
4558 return DepKind;
4559}
4560
4561/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4562static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4563 QualType &FlagsTy) {
4564 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4565 if (KmpDependInfoTy.isNull()) {
4566 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4567 KmpDependInfoRD->startDefinition();
4568 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4569 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4570 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4571 KmpDependInfoRD->completeDefinition();
4572 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
4573 }
4574}
4575
4576std::pair<llvm::Value *, LValue>
4577CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4578 SourceLocation Loc) {
4579 ASTContext &C = CGM.getContext();
4580 QualType FlagsTy;
4581 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4582 RecordDecl *KmpDependInfoRD =
4583 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4584 LValue Base = CGF.EmitLoadOfPointerLValue(
4585 DepobjLVal.getAddress(CGF),
4586 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4587 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4588 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4589 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4590 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4591 Base.getTBAAInfo());
4592 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4593 Addr.getPointer(),
4594 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4595 LValue NumDepsBase = CGF.MakeAddrLValue(
4596 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4597 Base.getBaseInfo(), Base.getTBAAInfo());
4598 // NumDeps = deps[i].base_addr;
4599 LValue BaseAddrLVal = CGF.EmitLValueForField(
4600 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4601 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4602 return std::make_pair(NumDeps, Base);
4603}
4604
4605static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4606 llvm::PointerUnion<unsigned *, LValue *> Pos,
4607 const OMPTaskDataTy::DependData &Data,
4608 Address DependenciesArray) {
4609 CodeGenModule &CGM = CGF.CGM;
4610 ASTContext &C = CGM.getContext();
4611 QualType FlagsTy;
4612 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4613 RecordDecl *KmpDependInfoRD =
4614 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4615 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4616
4617 OMPIteratorGeneratorScope IteratorScope(
4618 CGF, cast_or_null<OMPIteratorExpr>(
4619 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4620 : nullptr));
4621 for (const Expr *E : Data.DepExprs) {
4622 llvm::Value *Addr;
4623 llvm::Value *Size;
4624 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4625 LValue Base;
4626 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4627 Base = CGF.MakeAddrLValue(
4628 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4629 } else {
4630 LValue &PosLVal = *Pos.get<LValue *>();
4631 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4632 Base = CGF.MakeAddrLValue(
4633 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Idx),
4634 DependenciesArray.getAlignment()),
4635 KmpDependInfoTy);
4636 }
4637 // deps[i].base_addr = &<Dependencies[i].second>;
4638 LValue BaseAddrLVal = CGF.EmitLValueForField(
4639 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4640 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4641 BaseAddrLVal);
4642 // deps[i].len = sizeof(<Dependencies[i].second>);
4643 LValue LenLVal = CGF.EmitLValueForField(
4644 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
4645 CGF.EmitStoreOfScalar(Size, LenLVal);
4646 // deps[i].flags = <Dependencies[i].first>;
4647 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);
4648 LValue FlagsLVal = CGF.EmitLValueForField(
4649 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
4650 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
4651 FlagsLVal);
4652 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4653 ++(*P);
4654 } else {
4655 LValue &PosLVal = *Pos.get<LValue *>();
4656 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4657 Idx = CGF.Builder.CreateNUWAdd(Idx,
4658 llvm::ConstantInt::get(Idx->getType(), 1));
4659 CGF.EmitStoreOfScalar(Idx, PosLVal);
4660 }
4661 }
4662}
4663
4664static SmallVector<llvm::Value *, 4>
4665emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4666 const OMPTaskDataTy::DependData &Data) {
4667 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4668, __PRETTY_FUNCTION__))
4668 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4668, __PRETTY_FUNCTION__))
;
4669 SmallVector<llvm::Value *, 4> Sizes;
4670 SmallVector<LValue, 4> SizeLVals;
4671 ASTContext &C = CGF.getContext();
4672 QualType FlagsTy;
4673 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4674 RecordDecl *KmpDependInfoRD =
4675 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4676 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4677 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4678 {
4679 OMPIteratorGeneratorScope IteratorScope(
4680 CGF, cast_or_null<OMPIteratorExpr>(
4681 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4682 : nullptr));
4683 for (const Expr *E : Data.DepExprs) {
4684 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4685 LValue Base = CGF.EmitLoadOfPointerLValue(
4686 DepobjLVal.getAddress(CGF),
4687 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4688 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4689 Base.getAddress(CGF), KmpDependInfoPtrT);
4690 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4691 Base.getTBAAInfo());
4692 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4693 Addr.getPointer(),
4694 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4695 LValue NumDepsBase = CGF.MakeAddrLValue(
4696 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4697 Base.getBaseInfo(), Base.getTBAAInfo());
4698 // NumDeps = deps[i].base_addr;
4699 LValue BaseAddrLVal = CGF.EmitLValueForField(
4700 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4701 llvm::Value *NumDeps =
4702 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4703 LValue NumLVal = CGF.MakeAddrLValue(
4704 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4705 C.getUIntPtrType());
4706 CGF.InitTempAlloca(NumLVal.getAddress(CGF),
4707 llvm::ConstantInt::get(CGF.IntPtrTy, 0));
4708 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4709 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4710 CGF.EmitStoreOfScalar(Add, NumLVal);
4711 SizeLVals.push_back(NumLVal);
4712 }
4713 }
4714 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4715 llvm::Value *Size =
4716 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4717 Sizes.push_back(Size);
4718 }
4719 return Sizes;
4720}
4721
4722static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4723 LValue PosLVal,
4724 const OMPTaskDataTy::DependData &Data,
4725 Address DependenciesArray) {
4726 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4727, __PRETTY_FUNCTION__))
4727 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-12~++20210125100614+2cdb34efdac5/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4727, __PRETTY_FUNCTION__))
;
4728 ASTContext &C = CGF.getContext();
4729 QualType FlagsTy;
4730 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4731 RecordDecl *KmpDependInfoRD =
4732 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4733 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4734 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4735 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4736 {
4737 OMPIteratorGeneratorScope IteratorScope(
4738 CGF, cast_or_null<OMPIteratorExpr>(
4739 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4740 : nullptr));
4741 for (unsigned I = 0, End = Data.DepExprs.size(); I < End; ++I) {
4742 const Expr *E = Data.DepExprs[I];
4743 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4744 LValue Base = CGF.EmitLoadOfPointerLValue(
4745 DepobjLVal.getAddress(CGF),
4746 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4747 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4748 Base.getAddress(CGF), KmpDependInfoPtrT);
4749 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4750 Base.getTBAAInfo());
4751
4752 // Get number of elements in a single depobj.
4753 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4754 Addr.getPointer(),
4755 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4756 LValue NumDepsBase = CGF.MakeAddrLValue(
4757 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4758 Base.getBaseInfo(), Base.getTBAAInfo());
4759 // NumDeps = deps[i].base_addr;
4760 LValue BaseAddrLVal = CGF.EmitLValueForField(
4761 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4762 llvm::Value *NumDeps =
4763 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4764
4765 // memcopy dependency data.
4766 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4767 ElSize,
4768 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4769 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4770 Address DepAddr =
4771 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Pos),
4772 DependenciesArray.getAlignment());
4773 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
4774
4775 // Increase pos.
4776 // pos += size;
4777 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4778 CGF.EmitStoreOfScalar(Add, PosLVal);
4779 }
4780 }
4781}
4782
4783std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4784 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4785 SourceLocation Loc) {
4786 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {
4787 return D.DepExprs.empty();
4788 }))
4789 return std::make_pair(nullptr, Address::invalid());
4790 // Process list of dependencies.
4791 ASTContext &C = CGM.getContext();
4792 Address DependenciesArray = Address::invalid();
4793 llvm::Value *NumOfElements = nullptr;
4794 unsigned NumDependencies = std::accumulate(
4795 Dependencies.begin(), Dependencies.end(), 0,
4796 [](unsigned V, const OMPTaskDataTy::DependData &D) {
4797 return D.DepKind == OMPC_DEPEND_depobj
4798 ? V
4799 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4800 });
4801 QualType FlagsTy;
4802 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4803 bool HasDepobjDeps = false;
4804 bool HasRegularWithIterators = false;
4805 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4806 llvm::Value *NumOfRegularWithIterators =
4807 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4808 // Calculate number of depobj dependecies and regular deps with the iterators.
4809 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4810 if (D.DepKind == OMPC_DEPEND_depobj) {
4811 SmallVector<llvm::Value *, 4> Sizes =
4812 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4813 for (llvm::Value *Size : Sizes) {
4814 NumOfDepobjElements =
4815 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4816 }
4817 HasDepobjDeps = true;
4818 continue;
4819 }
4820 // Include number of iterations, if any.
4821 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {
4822 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4823 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4824 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4825 NumOfRegularWithIterators =
4826 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz);
4827 }
4828 HasRegularWithIterators = true;
4829 continue;
4830 }
4831 }
4832
4833 QualType KmpDependInfoArrayTy;
4834 if (HasDepobjDeps || HasRegularWithIterators) {
4835 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,
4836 /*isSigned=*/false);
4837 if (HasDepobjDeps) {
4838 NumOfElements =
4839 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4840 }
4841 if (HasRegularWithIterators) {
4842 NumOfElements =
4843 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4844 }
4845 OpaqueValueExpr OVE(Loc,
4846 C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4847 VK_RValue);
4848 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4849 RValue::get(NumOfElements));
4850 KmpDependInfoArrayTy =
4851 C.getVariableArrayType(KmpDependInfoTy, &OVE, ArrayType::Normal,
4852 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4853 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4854 // Properly emit variable-sized array.
4855 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,
4856 ImplicitParamDecl::Other);
4857 CGF.EmitVarDecl(*PD);
4858 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4859 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4860 /*isSigned=*/false);
4861 } else {
4862 KmpDependInfoArrayTy = C.getConstantArrayType(
4863 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,
4864 ArrayType::Normal, /*IndexTypeQuals=*/0);
4865 DependenciesArray =
4866 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4867 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4868 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,
4869 /*isSigned=*/false);
4870 }
4871 unsigned Pos = 0;
4872 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4873 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4874 Dependencies[I].IteratorExpr)
4875 continue;
4876 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4877 DependenciesArray);
4878 }
4879 // Copy regular dependecies with iterators.
4880 LValue PosLVal = CGF.MakeAddrLValue(
4881 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4882 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4883 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4884 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4885 !Dependencies[I].IteratorExpr)
4886 continue;
4887 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4888 DependenciesArray);
4889 }
4890 // Copy final depobj arrays without iterators.
4891 if (HasDepobjDeps) {
4892 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4893 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4894 continue;
4895 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4896 DependenciesArray);
4897 }
4898 }
4899 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4900 DependenciesArray, CGF.VoidPtrTy);
4901 return std::make_pair(NumOfElements, DependenciesArray);
4902}
4903
4904Address CGOpenMPRuntime::emitDepobjDependClause(
4905 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4906 SourceLocation Loc) {
4907 if (Dependencies.DepExprs.empty())
4908 return Address::invalid();
4909 // Process list of dependencies.
4910 ASTContext &C = CGM.getContext();
4911 Address DependenciesArray = Address::invalid();
4912 unsigned NumDependencies = Dependencies.DepExprs.size();
4913 QualType FlagsTy;
4914 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4915 RecordDecl *KmpDependInfoRD =
4916 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4917
4918 llvm::Value *Size;
4919 // Define type kmp_depend_info[<Dependencies.size()>];
4920 // For depobj reserve one extra element to store the number of elements.
4921 // It is required to handle depobj(x) update(in) construct.
4922 // kmp_depend_info[<Dependencies.size()>] deps;
4923 llvm::Value *NumDepsVal;
4924 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4925 if (const auto *IE =
4926 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {
4927 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4928 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4929 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4930 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4931 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4932 }
4933 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4934 NumDepsVal);
4935 CharUnits SizeInBytes =
4936 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4937 llvm::Value *RecSize = CGM.getSize(SizeInBytes);
4938 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4939 NumDepsVal =
4940 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4941 } else {
4942 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4943 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4944 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
4945 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4946 Size = CGM.getSize(Sz.alignTo(Align));
4947 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4948 }
4949 // Need to allocate on the dynamic memory.
4950 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4951 // Use default allocator.
4952 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4953 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4954
4955 llvm::Value *Addr =
4956 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4957 CGM.getModule(), OMPRTL___kmpc_alloc),
4958 Args, ".dep.arr.addr");
4959 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4960 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo());
4961 DependenciesArray = Address(Addr, Align);
4962 // Write number of elements in the first element of array for depobj.
4963 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4964 // deps[i].base_addr = NumDependencies;
4965 LValue BaseAddrLVal = CGF.EmitLValueForField(
4966 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4967 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4968 llvm::PointerUnion<unsigned *, LValue *> Pos;
4969 unsigned Idx = 1;
4970 LValue PosLVal;
4971 if (Dependencies.IteratorExpr) {
4972 PosLVal = CGF.MakeAddrLValue(
4973 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4974 C.getSizeType());
4975 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4976 /*IsInit=*/true);
4977 Pos = &PosLVal;
4978 } else {
4979 Pos = &Idx;
4980 }
4981 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4982 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4983 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy);
4984 return DependenciesArray;
4985}
4986
4987void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4988 SourceLocation Loc) {
4989 ASTContext &C = CGM.getContext();
4990 QualType FlagsTy;
4991 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4992 LValue Base = CGF.EmitLoadOfPointerLValue(
4993 DepobjLVal.getAddress(CGF),
4994 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4995 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4996 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4997 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4998 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4999 Addr.getPointer(),
5000 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
5001 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
5002 CGF.VoidPtrTy);
5003 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5004 // Use default allocator.
5005 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5006 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
5007
5008 // _kmpc_free(gtid, addr, nullptr);
5009 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5010 CGM.getModule(), OMPRTL___kmpc_free),
5011 Args);
5012}
5013
5014void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
5015 OpenMPDependClauseKind NewDepKind,
5016 SourceLocation Loc) {
5017 ASTContext &C = CGM.getContext();
5018 QualType FlagsTy;
5019 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5020 RecordDecl *KmpDependInfoRD =
5021 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5022 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5023 llvm::Value *NumDeps;
5024 LValue Base;
5025 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
5026
5027 Address Begin = Base.getAddress(CGF);
5028 // Cast from pointer to array type to pointer to single element.
5029 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps);
5030 // The basic structure here is a while-do loop.
5031 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
5032 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
5033 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5034 CGF.EmitBlock(BodyBB);
5035 llvm::PHINode *ElementPHI =
5036 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
5037 ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
5038 Begin = Address(ElementPHI, Begin.getAlignment());
5039 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
5040 Base.getTBAAInfo());
5041 // deps[i].flags = NewDepKind;
5042 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
5043 LValue FlagsLVal = CGF.EmitLValueForField(
5044 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5045 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5046 FlagsLVal);
5047
5048 // Shift the address forward by one element.
5049 Address ElementNext =
5050 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
5051 ElementPHI->addIncoming(ElementNext.getPointer(),
5052 CGF.Builder.GetInsertBlock());
5053 llvm::Value *IsEmpty =
5054 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
5055 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5056 // Done.
5057 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5058}
5059
5060void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5061 const OMPExecutableDirective &D,
5062 llvm::Function *TaskFunction,
5063 QualType SharedsTy, Address Shareds,
5064 const Expr *IfCond,
5065 const OMPTaskDataTy &Data) {
5066 if (!CGF.HaveInsertPoint())
5067 return;
5068
5069 TaskResultTy Result =
5070 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5071 llvm::Value *NewTask = Result.NewTask;
5072 llvm::Function *TaskEntry = Result.TaskEntry;
5073 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5074 LValue TDBase = Result.TDBase;
5075 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5076 // Process list of dependences.
5077 Address DependenciesArray = Address::invalid();
5078 llvm::Value *NumOfElements;
5079 std::tie(NumOfElements, DependenciesArray) =
5080 emitDependClause(CGF, Data.Dependences, Loc);
5081
5082 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5083 // libcall.
5084 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5085 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5086 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5087 // list is not empty
5088 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5089 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5090 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5091 llvm::Value *DepTaskArgs[7];
5092 if (!Data.Dependences.empty()) {
5093 DepTaskArgs[0] = UpLoc;
5094 DepTaskArgs[1] = ThreadID;
5095 DepTaskArgs[2] = NewTask;
5096 DepTaskArgs[3] = NumOfElements;
5097 DepTaskArgs[4] = DependenciesArray.getPointer();
5098 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5099 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5100 }
5101 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
5102 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5103 if (!Data.Tied) {
5104 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5105 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5106 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5107 }
5108 if (!Data.Dependences.empty()) {
5109 CGF.EmitRuntimeCall(
5110 OMPBuilder.getOrCreateRuntimeFunction(
5111 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
5112 DepTaskArgs);
5113 } else {
5114 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5115 CGM.getModule(), OMPRTL___kmpc_omp_task),
5116 TaskArgs);
5117 }
5118 // Check if parent region is untied and build return for untied task;
5119 if (auto *Region =
5120 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5121 Region->emitUntiedSwitch(CGF);
5122 };
5123
5124 llvm::Value *DepWaitTaskArgs[6];
5125 if (!Data.Dependences.empty()) {
5126 DepWaitTaskArgs[0] = UpLoc;
5127 DepWaitTaskArgs[1] = ThreadID;
5128 DepWaitTaskArgs[2] = NumOfElements;
5129 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5130 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5131 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5132 }
5133 auto &M = CGM.getModule();
5134 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
5135 TaskEntry, &Data, &DepWaitTaskArgs,
5136 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5137 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5138 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5139 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5140 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5141 // is specified.
5142 if (!Data.Dependences.empty())
5143 CGF.EmitRuntimeCall(
5144 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_wait_deps),
5145 DepWaitTaskArgs);
5146 // Call proxy_task_entry(gtid, new_task);
5147 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5148 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5149 Action.Enter(CGF);
5150 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5151 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5152 OutlinedFnArgs);
5153 };
5154
5155 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5156 // kmp_task_t *new_task);
5157 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5158 // kmp_task_t *new_task);
5159 RegionCodeGenTy RCG(CodeGen);
5160 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
5161 M, OMPRTL___kmpc_omp_task_begin_if0),
5162 TaskArgs,
5163 OMPBuilder.getOrCreateRuntimeFunction(
5164 M, OMPRTL___kmpc_omp_task_complete_if0),
5165 TaskArgs);
5166 RCG.setAction(Action);
5167 RCG(CGF);
5168 };
5169
5170 if (IfCond) {
5171 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5172 } else {
5173 RegionCodeGenTy ThenRCG(ThenCodeGen);
5174 ThenRCG(CGF);
5175 }
5176}
5177
5178void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5179 const OMPLoopDirective &D,
5180 llvm::Function *TaskFunction,
5181 QualType SharedsTy, Address Shareds,
5182 const Expr *IfCond,
5183 const OMPTaskDataTy &Data) {
5184 if (!CGF.HaveInsertPoint())
5185 return;
5186 TaskResultTy Result =
5187 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5188 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5189 // libcall.
5190 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5191 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5192 // sched, kmp_uint64 grainsize, void *task_dup);
5193 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5194 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5195 llvm::Value *IfVal;
5196 if (IfCond) {
5197 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5198 /*isSigned=*/true);
5199 } else {
5200 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5201 }
5202
5203 LValue LBLVal = CGF.EmitLValueForField(
5204 Result.TDBase,
5205 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5206 const auto *LBVar =
5207 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5208 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5209 LBLVal.getQuals(),
5210 /*IsInitializer=*/true);
5211 LValue UBLVal = CGF.EmitLValueForField(
5212 Result.TDBase,
5213 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5214 const auto *UBVar =
5215 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5216 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5217 UBLVal.getQuals(),
5218 /*IsInitializer=*/true);
5219 LValue StLVal = CGF.EmitLValueForField(
5220 Result.TDBase,
5221 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5222 const auto *StVar =
5223 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5224 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5225 StLVal.getQuals(),
5226 /*IsInitializer=*/true);
5227 // Store reductions address.
5228 LValue RedLVal = CGF.EmitLValueForField(
5229 Result.TDBase,
5230 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5231 if (Data.Reductions) {
5232 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5233 } else {
5234 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5235 CGF.getContext().VoidPtrTy);
5236 }
5237 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5238 llvm::Value *TaskArgs[] = {
5239 UpLoc,
5240 ThreadID,
5241 Result.NewTask,
5242 IfVal,
5243 LBLVal.getPointer(CGF),
5244 UBLVal.getPointer(CGF),
5245 CGF.EmitLoadOfScalar(StLVal, Loc),
5246 llvm::ConstantInt::getSigned(
5247 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5248 llvm::ConstantInt::getSigned(
5249 CGF.IntTy, Data.Schedule.getPointer()
5250 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5251 : NoSchedule),
5252 Data.Schedule.getPointer()
5253 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5254 /*isSigned=*/false)
5255 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5256 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5257 Result.TaskDupFn, CGF.VoidPtrTy)
5258 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5259 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5260 CGM.getModule(), OMPRTL___kmpc_taskloop),
5261 TaskArgs);
5262}
5263
5264/// Emit reduction operation for each element of array (required for
5265/// array sections) LHS op = RHS.
5266/// \param Type Type of array.
5267/// \param LHSVar Variable on the left side of the reduction operation
5268/// (references element of array in original variable).
5269/// \param RHSVar Variable on the right side of the reduction operation
5270/// (references element of array in original variable).
5271/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5272/// RHSVar.
5273static void EmitOMPAggregateReduction(
5274 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5275 const VarDecl *RHSVar,
5276 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5277 const Expr *, const Expr *)> &RedOpGen,
5278 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5279 const Expr *UpExpr = nullptr) {
5280 // Perform element-by-element initialization.
5281 QualType ElementTy;
5282 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5283 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5284
5285 // Drill down to the base element type on both arrays.
5286 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5287 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5288
5289 llvm::Value *RHSBegin = RHSAddr.getPointer();
5290 llvm::Value *LHSBegin = LHSAddr.getPointer();
5291 // Cast from pointer to array type to pointer to single element.
5292 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5293 // The basic structure here is a while-do loop.
5294 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5295 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5296 llvm::Value *IsEmpty =
5297 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5298 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5299
5300 // Enter the loop body, making that address the current address.
5301 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5302 CGF.EmitBlock(BodyBB);
5303
5304 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5305
5306 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5307 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5308 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5309 Address RHSElementCurrent =
5310 Address(RHSElementPHI,
5311 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5312
5313 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5314 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5315 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5316 Address LHSElementCurrent =
5317 Address(LHSElementPHI,
5318 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5319
5320 // Emit copy.
5321 CodeGenFunction::OMPPrivateScope Scope(CGF);
5322 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5323 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5324 Scope.Privatize();
5325 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5326 Scope.ForceCleanup();
5327
5328 // Shift the address forward by one element.
5329 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5330 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5331 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5332 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5333 // Check whether we've reached the end.
5334 llvm::Value *Done =
5335 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5336 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5337 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5338 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5339
5340 // Done.
5341 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5342}
5343
5344/// Emit reduction combiner. If the combiner is a simple expression emit it as
5345/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5346/// UDR combiner function.
5347static void emitReductionCombiner(CodeGenFunction &CGF,
5348 const Expr *ReductionOp) {
5349 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5350 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5351 if (const auto *DRE =
5352 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5353 if (const auto *DRD =
5354 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5355 std::pair<llvm::Function *, llvm::Function *> Reduction =
5356 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5357 RValue Func = RValue::get(Reduction.first);
5358 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5359 CGF.EmitIgnoredExpr(ReductionOp);
5360 return;
5361 }
5362 CGF.EmitIgnoredExpr(ReductionOp);
5363}
5364
5365llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5366 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5367 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5368 ArrayRef<const Expr *> ReductionOps) {
5369 ASTContext &C = CGM.getContext();
5370
5371 // void reduction_func(void *LHSArg, void *RHSArg);
5372 FunctionArgList Args;
5373 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5374 ImplicitParamDecl::Other);
5375 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5376 ImplicitParamDecl::Other);
5377 Args.push_back(&LHSArg);
5378 Args.push_back(&RHSArg);
5379 const auto &CGFI =
5380 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5381 std::string Name = getName({"omp", "reduction", "reduction_func"});
5382 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5383 llvm::GlobalValue::InternalLinkage, Name,
5384 &CGM.getModule());
5385 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5386 Fn->setDoesNotRecurse();
5387 CodeGenFunction CGF(CGM);
5388 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5389
5390 // Dst = (void*[n])(LHSArg);
5391 // Src = (void*[n])(RHSArg);
5392 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5393 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5394 ArgsType), CGF.getPointerAlign());
5395 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5396 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5397 ArgsType), CGF.getPointerAlign());
5398
5399 // ...
5400 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5401 // ...
5402 CodeGenFunction::OMPPrivateScope Scope(CGF);
5403 auto IPriv = Privates.begin();
5404 unsigned Idx = 0;
5405 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5406 const auto *RHSVar =
5407 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5408 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5409 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5410 });
5411 const auto *LHSVar =
5412 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5413 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5414 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5415 });
5416 QualType PrivTy = (*IPriv)->getType();
5417 if (PrivTy->isVariablyModifiedType()) {
5418 // Get array size and emit VLA type.
5419 ++Idx;
5420 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5421 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5422 const VariableArrayType *VLA =
5423 CGF.getContext().getAsVariableArrayType(PrivTy);
5424 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5425 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5426 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5427 CGF.EmitVariablyModifiedType(PrivTy);
5428 }
5429 }
5430 Scope.Privatize();
5431 IPriv = Privates.begin();
5432 auto ILHS = LHSExprs.begin();
5433 auto IRHS = RHSExprs.begin();
5434 for (const Expr *E : ReductionOps) {
5435 if ((*IPriv)->getType()->isArrayType()) {
5436 // Emit reduction for array section.
5437 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5438 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5439 EmitOMPAggregateReduction(
5440 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5441 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5442 emitReductionCombiner(CGF, E);
5443 });
5444 } else {
5445 // Emit reduction for array subscript or single variable.
5446 emitReductionCombiner(CGF, E);
5447 }
5448 ++IPriv;
5449 ++ILHS;
5450 ++IRHS;
5451 }
5452 Scope.ForceCleanup();
5453 CGF.FinishFunction();
5454 return Fn;
5455}
5456
5457void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5458 const Expr *ReductionOp,
5459 const Expr *PrivateRef,
5460 const DeclRefExpr *LHS,
5461 const DeclRefExpr *RHS) {
5462 if (PrivateRef->getType()->isArrayType()) {
5463 // Emit reduction for array section.
5464 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5465 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5466 EmitOMPAggregateReduction(
5467 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5468 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5469 emitReductionCombiner(CGF, ReductionOp);
5470 });
5471 } else {
5472 // Emit reduction for array subscript or single variable.
5473 emitReductionCombiner(CGF, ReductionOp);
5474 }
5475}
5476
5477void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5478 ArrayRef<const Expr *> Privates,
5479 ArrayRef<const Expr *> LHSExprs,
5480 ArrayRef<const Expr *> RHSExprs,
5481 ArrayRef<const Expr *> ReductionOps,
5482 ReductionOptionsTy Options) {
5483 if (!CGF.HaveInsertPoint())
5484 return;
5485
5486 bool WithNowait = Options.WithNowait;
5487 bool SimpleReduction = Options.SimpleReduction;
5488
5489 // Next code should be emitted for reduction:
5490 //
5491 // static kmp_critical_name lock = { 0 };
5492 //
5493 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5494 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5495 // ...
5496 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5497 // *(Type<n>-1*)rhs[<n>-1]);
5498 // }
5499 //
5500 // ...
5501 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5502 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5503 // RedList, reduce_func, &<lock>)) {
5504 // case 1:
5505 // ...
5506 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5507 // ...
5508 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5509 // break;
5510 // case 2:
5511 // ...
5512 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5513 // ...
5514 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5515 // break;
5516 // default:;
5517 // }
5518 //
5519 // if SimpleReduction is true, only the next code is generated:
5520 // ...
5521 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5522 // ...
5523
5524 ASTContext &C = CGM.getContext();
5525
5526 if (SimpleReduction) {
5527 CodeGenFunction::RunCleanupsScope Scope(CGF);
5528 auto IPriv = Privates.begin();
5529 auto ILHS = LHSExprs.begin();
5530 auto IRHS = RHSExprs.begin();
5531 for (const Expr *E : ReductionOps) {
5532 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5533 cast<DeclRefExpr>(*IRHS));
5534 ++IPriv;
5535 ++ILHS;
5536 ++IRHS;
5537 }
5538 return;
5539 }
5540
5541 // 1. Build a list of reduction variables.
5542 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5543 auto Size = RHSExprs.size();
5544 for (const Expr *E : Privates) {
5545 if (E->getType()->isVariablyModifiedType())
5546 // Reserve place for array size.
5547 ++Size;
5548 }
5549 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5550 QualType ReductionArrayTy =
5551 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5552 /*IndexTypeQuals=*/0);
5553 Address ReductionList =
5554 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5555 auto IPriv = Privates.begin();
5556 unsigned Idx = 0;
5557 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5558 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5559 CGF.Builder.CreateStore(
5560 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5561 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5562 Elem);
5563 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5564 // Store array size.
5565 ++Idx;
5566 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5567 llvm::Value *Size = CGF.Builder.CreateIntCast(
5568 CGF.getVLASize(
5569 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5570 .NumElts,
5571 CGF.SizeTy, /*isSigned=*/false);
5572 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5573 Elem);
5574 }
5575 }
5576
5577 // 2. Emit reduce_func().
5578 llvm::Function *ReductionFn = emitReductionFunction(
5579 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5580 LHSExprs, RHSExprs, ReductionOps);
5581
5582 // 3. Create static kmp_critical_name lock = { 0 };
5583 std::string Name = getName({"reduction"});
5584 llvm::Value *Lock = getCriticalRegionLock(Name);
5585
5586 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5587 // RedList, reduce_func, &<lock>);
5588 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5589 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5590 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5591 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5592 ReductionList.getPointer(), CGF.VoidPtrTy);
5593 llvm::Value *Args[] = {
5594 IdentTLoc, // ident_t *<loc>
5595 ThreadId, // i32 <gtid>
5596 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5597 ReductionArrayTySize, // size_type sizeof(RedList)
5598 RL, // void *RedList
5599 ReductionFn, // void (*) (void *, void *) <reduce_func>
5600 Lock // kmp_critical_name *&<lock>
5601 };
5602 llvm::Value *Res = CGF.EmitRuntimeCall(
5603 OMPBuilder.getOrCreateRuntimeFunction(
5604 CGM.getModule(),
5605 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5606 Args);
5607
5608 // 5. Build switch(res)
5609 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5610 llvm::SwitchInst *SwInst =
5611 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5612
5613 // 6. Build case 1:
5614 // ...
5615 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5616 // ...
5617 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5618 // break;
5619 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5620 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5621 CGF.EmitBlock(Case1BB);
5622
5623 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5624 llvm::Value *EndArgs[] = {
5625 IdentTLoc, // ident_t *<loc>
5626 ThreadId, // i32 <gtid>
5627 Lock // kmp_critical_name *&<lock>
5628 };
5629 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5630 CodeGenFunction &CGF, PrePostActionTy &Action) {
5631 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5632 auto IPriv = Privates.begin();
5633 auto ILHS = LHSExprs.begin();
5634 auto IRHS = RHSExprs.begin();
5635 for (const Expr *E : ReductionOps) {
5636 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5637 cast<DeclRefExpr>(*IRHS));
5638 ++IPriv;
5639 ++ILHS;
5640 ++IRHS;
5641 }
5642 };
5643 RegionCodeGenTy RCG(CodeGen);
5644 CommonActionTy Action(
5645 nullptr, llvm::None,
5646 OMPBuilder.getOrCreateRuntimeFunction(
5647 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5648 : OMPRTL___kmpc_end_reduce),
5649 EndArgs);
5650 RCG.setAction(Action);
5651 RCG(CGF);
5652
5653 CGF.EmitBranch(DefaultBB);
5654
5655 // 7. Build case 2:
5656 // ...
5657 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5658 // ...
5659 // break;
5660 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5661 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5662 CGF.EmitBlock(Case2BB);
5663
5664 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5665 CodeGenFunction &CGF, PrePostActionTy &Action) {
5666 auto ILHS = LHSExprs.begin();
5667 auto IRHS = RHSExprs.begin();
5668 auto IPriv = Privates.begin();
5669 for (const Expr *E : ReductionOps) {
5670 const Expr *XExpr = nullptr;
5671 const Expr *EExpr = nullptr;
5672 const Expr *UpExpr = nullptr;
5673 BinaryOperatorKind BO = BO_Comma;
5674 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5675 if (BO->getOpcode() == BO_Assign) {
5676 XExpr = BO->getLHS();
5677 UpExpr = BO->getRHS();
5678 }
5679 }
5680 // Try to emit update expression as a simple atomic.
5681 const Expr *RHSExpr = UpExpr;
5682 if (RHSExpr) {
5683 // Analyze RHS part of the whole expression.
5684 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5685 RHSExpr->IgnoreParenImpCasts())) {
5686 // If this is a conditional operator, analyze its condition for
5687 // min/max reduction operator.
5688 RHSExpr = ACO->getCond();
5689 }
5690 if (const auto *BORHS =
5691 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5692 EExpr = BORHS->getRHS();
5693 BO = BORHS->getOpcode();
5694 }
5695 }
5696 if (XExpr) {
5697 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5698 auto &&AtomicRedGen = [BO, VD,
5699 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5700 const Expr *EExpr, const Expr *UpExpr) {
5701 LValue X = CGF.EmitLValue(XExpr);
5702 RValue E;
5703 if (EExpr)
5704 E = CGF.EmitAnyExpr(EExpr);
5705 CGF.EmitOMPAtomicSimpleUpdateExpr(
5706 X, E, BO, /*IsXLHSInRHSPart=*/true,
5707 llvm::AtomicOrdering::Monotonic, Loc,
5708 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5709 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5710 PrivateScope.addPrivate(
5711 VD, [&CGF, VD, XRValue, Loc]() {
5712 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5713 CGF.emitOMPSimpleStore(
5714 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5715 VD->getType().getNonReferenceType(), Loc);
5716 return LHSTemp;
5717 });
5718 (void)PrivateScope.Privatize();
5719 return CGF.EmitAnyExpr(UpExpr);
5720 });
5721 };
5722 if ((*IPriv)->getType()->isArrayType()) {
5723 // Emit atomic reduction for array section.
5724 const auto *RHSVar =
5725 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5726 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5727 AtomicRedGen, XExpr, EExpr, UpExpr);
5728 } else {
5729 // Emit atomic reduction for array subscript or single variable.
5730 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5731 }
5732 } else {
5733 // Emit as a critical region.
5734 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5735 const Expr *, const Expr *) {
5736 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5737 std::string Name = RT.getName({"atomic_reduction"});
5738 RT.emitCriticalRegion(
5739 CGF, Name,
5740 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5741 Action.Enter(CGF);
5742 emitReductionCombiner(CGF, E);
5743 },
5744 Loc);
5745 };
5746 if ((*IPriv)->getType()->isArrayType()) {
5747 const auto *LHSVar =
5748 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5749 const auto *RHSVar =
5750 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5751 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5752 CritRedGen);
5753 } else {
5754 CritRedGen(CGF, nullptr, nullptr, nullptr);
5755 }
5756 }
5757 ++ILHS;
5758 ++IRHS;
5759 ++IPriv;
5760 }
5761 };
5762 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5763 if (!WithNowait) {
5764 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5765 llvm::Value *EndArgs[] = {
5766 IdentTLoc, // ident_t *<loc>
5767 ThreadId, // i32 <gtid>
5768 Lock // kmp_critical_name *&<lock>
5769 };
5770 CommonActionTy Action(nullptr, llvm::None,
5771 OMPBuilder.getOrCreateRuntimeFunction(
5772 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5773 EndArgs);
5774 AtomicRCG.setAction(Action);
5775 AtomicRCG(CGF);
5776 } else {
5777 AtomicRCG(CGF);
5778 }
5779
5780 CGF.EmitBranch(DefaultBB);
5781 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5782}
5783
5784/// Generates unique name for artificial threadprivate variables.
5785/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5786static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5787 const Expr *Ref) {
5788 SmallString<256> Buffer;
5789 llvm::raw_svector_ostream Out(Buffer);
5790 const clang::DeclRefExpr *DE;
5791 const VarDecl *D = ::getBaseDecl(Ref, DE);
5792 if (!D)
5793 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5794 D = D->getCanonicalDecl();
5795 std::string Name = CGM.getOpenMPRuntime().getName(
5796 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5797 Out << Prefix << Name << "_"
5798 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5799 return std::string(Out.str());
5800}
5801
5802/// Emits reduction initializer function:
5803/// \code
5804/// void @.red_init(void* %arg, void* %orig) {
5805/// %0 = bitcast void* %arg to <type>*
5806/// store <type> <init>, <type>* %0
5807/// ret void
5808/// }
5809/// \endcode
5810static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5811 SourceLocation Loc,
5812 ReductionCodeGen &RCG, unsigned N) {
5813 ASTContext &C = CGM.getContext();
5814 QualType VoidPtrTy = C.VoidPtrTy;
5815 VoidPtrTy.addRestrict();
5816 FunctionArgList Args;
5817 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, VoidPtrTy,
5818 ImplicitParamDecl::Other);
5819 ImplicitParamDecl ParamOrig(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, VoidPtrTy,
5820 ImplicitParamDecl::Other);
5821 Args.emplace_back(&Param);
5822 Args.emplace_back(&ParamOrig);
5823 const