Bug Summary

File:clang/lib/CodeGen/CGOpenMPRuntime.cpp
Warning:line 8004, column 9
2nd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGOpenMPRuntime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/build-llvm/tools/clang/lib/CodeGen -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/include -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-03-02-022427-27315-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp

/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp

1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "CGCXXABI.h"
15#include "CGCleanup.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "clang/Basic/BitmaskEnum.h"
24#include "clang/Basic/FileManager.h"
25#include "clang/Basic/OpenMPKinds.h"
26#include "clang/Basic/SourceManager.h"
27#include "clang/CodeGen/ConstantInitBuilder.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/SetOperations.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/Bitcode/BitcodeReader.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/Value.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40#include <numeric>
41
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46namespace {
47/// Base class for handling code generation inside OpenMP regions.
48class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
49public:
50 /// Kinds of OpenMP regions used in codegen.
51 enum CGOpenMPRegionKind {
52 /// Region with outlined function for standalone 'parallel'
53 /// directive.
54 ParallelOutlinedRegion,
55 /// Region with outlined function for standalone 'task' directive.
56 TaskOutlinedRegion,
57 /// Region for constructs that do not require function outlining,
58 /// like 'for', 'sections', 'atomic' etc. directives.
59 InlinedRegion,
60 /// Region with outlined function for standalone 'target' directive.
61 TargetRegion,
62 };
63
64 CGOpenMPRegionInfo(const CapturedStmt &CS,
65 const CGOpenMPRegionKind RegionKind,
66 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
67 bool HasCancel)
68 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
69 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
70
71 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
72 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
73 bool HasCancel)
74 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
75 Kind(Kind), HasCancel(HasCancel) {}
76
77 /// Get a variable or parameter for storing global thread id
78 /// inside OpenMP construct.
79 virtual const VarDecl *getThreadIDVariable() const = 0;
80
81 /// Emit the captured statement body.
82 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
83
84 /// Get an LValue for the current ThreadID variable.
85 /// \return LValue for thread id variable. This LValue always has type int32*.
86 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
87
88 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
89
90 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
91
92 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
93
94 bool hasCancel() const { return HasCancel; }
95
96 static bool classof(const CGCapturedStmtInfo *Info) {
97 return Info->getKind() == CR_OpenMP;
98 }
99
100 ~CGOpenMPRegionInfo() override = default;
101
102protected:
103 CGOpenMPRegionKind RegionKind;
104 RegionCodeGenTy CodeGen;
105 OpenMPDirectiveKind Kind;
106 bool HasCancel;
107};
108
109/// API for captured statement code generation in OpenMP constructs.
110class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
111public:
112 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
113 const RegionCodeGenTy &CodeGen,
114 OpenMPDirectiveKind Kind, bool HasCancel,
115 StringRef HelperName)
116 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
117 HasCancel),
118 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
119 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 119, __PRETTY_FUNCTION__))
;
120 }
121
122 /// Get a variable or parameter for storing global thread id
123 /// inside OpenMP construct.
124 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
125
126 /// Get the name of the capture helper.
127 StringRef getHelperName() const override { return HelperName; }
128
129 static bool classof(const CGCapturedStmtInfo *Info) {
130 return CGOpenMPRegionInfo::classof(Info) &&
131 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
132 ParallelOutlinedRegion;
133 }
134
135private:
136 /// A variable or parameter storing global thread id for OpenMP
137 /// constructs.
138 const VarDecl *ThreadIDVar;
139 StringRef HelperName;
140};
141
142/// API for captured statement code generation in OpenMP constructs.
143class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
144public:
145 class UntiedTaskActionTy final : public PrePostActionTy {
146 bool Untied;
147 const VarDecl *PartIDVar;
148 const RegionCodeGenTy UntiedCodeGen;
149 llvm::SwitchInst *UntiedSwitch = nullptr;
150
151 public:
152 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
153 const RegionCodeGenTy &UntiedCodeGen)
154 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
155 void Enter(CodeGenFunction &CGF) override {
156 if (Untied) {
157 // Emit task switching point.
158 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
159 CGF.GetAddrOfLocalVar(PartIDVar),
160 PartIDVar->getType()->castAs<PointerType>());
161 llvm::Value *Res =
162 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
163 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
164 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
165 CGF.EmitBlock(DoneBB);
166 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
167 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
168 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
169 CGF.Builder.GetInsertBlock());
170 emitUntiedSwitch(CGF);
171 }
172 }
173 void emitUntiedSwitch(CodeGenFunction &CGF) const {
174 if (Untied) {
175 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
176 CGF.GetAddrOfLocalVar(PartIDVar),
177 PartIDVar->getType()->castAs<PointerType>());
178 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
179 PartIdLVal);
180 UntiedCodeGen(CGF);
181 CodeGenFunction::JumpDest CurPoint =
182 CGF.getJumpDestInCurrentScope(".untied.next.");
183 CGF.EmitBranch(CGF.ReturnBlock.getBlock());
184 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
185 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
186 CGF.Builder.GetInsertBlock());
187 CGF.EmitBranchThroughCleanup(CurPoint);
188 CGF.EmitBlock(CurPoint.getBlock());
189 }
190 }
191 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
192 };
193 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
194 const VarDecl *ThreadIDVar,
195 const RegionCodeGenTy &CodeGen,
196 OpenMPDirectiveKind Kind, bool HasCancel,
197 const UntiedTaskActionTy &Action)
198 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
199 ThreadIDVar(ThreadIDVar), Action(Action) {
200 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.")((ThreadIDVar != nullptr && "No ThreadID in OpenMP region."
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 200, __PRETTY_FUNCTION__))
;
201 }
202
203 /// Get a variable or parameter for storing global thread id
204 /// inside OpenMP construct.
205 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
206
207 /// Get an LValue for the current ThreadID variable.
208 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
209
210 /// Get the name of the capture helper.
211 StringRef getHelperName() const override { return ".omp_outlined."; }
212
213 void emitUntiedSwitch(CodeGenFunction &CGF) override {
214 Action.emitUntiedSwitch(CGF);
215 }
216
217 static bool classof(const CGCapturedStmtInfo *Info) {
218 return CGOpenMPRegionInfo::classof(Info) &&
219 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
220 TaskOutlinedRegion;
221 }
222
223private:
224 /// A variable or parameter storing global thread id for OpenMP
225 /// constructs.
226 const VarDecl *ThreadIDVar;
227 /// Action for emitting code for untied tasks.
228 const UntiedTaskActionTy &Action;
229};
230
231/// API for inlined captured statement code generation in OpenMP
232/// constructs.
233class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
234public:
235 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
236 const RegionCodeGenTy &CodeGen,
237 OpenMPDirectiveKind Kind, bool HasCancel)
238 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
239 OldCSI(OldCSI),
240 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
241
242 // Retrieve the value of the context parameter.
243 llvm::Value *getContextValue() const override {
244 if (OuterRegionInfo)
245 return OuterRegionInfo->getContextValue();
246 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 246)
;
247 }
248
249 void setContextValue(llvm::Value *V) override {
250 if (OuterRegionInfo) {
251 OuterRegionInfo->setContextValue(V);
252 return;
253 }
254 llvm_unreachable("No context value for inlined OpenMP region")::llvm::llvm_unreachable_internal("No context value for inlined OpenMP region"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 254)
;
255 }
256
257 /// Lookup the captured field decl for a variable.
258 const FieldDecl *lookup(const VarDecl *VD) const override {
259 if (OuterRegionInfo)
260 return OuterRegionInfo->lookup(VD);
261 // If there is no outer outlined region,no need to lookup in a list of
262 // captured variables, we can use the original one.
263 return nullptr;
264 }
265
266 FieldDecl *getThisFieldDecl() const override {
267 if (OuterRegionInfo)
268 return OuterRegionInfo->getThisFieldDecl();
269 return nullptr;
270 }
271
272 /// Get a variable or parameter for storing global thread id
273 /// inside OpenMP construct.
274 const VarDecl *getThreadIDVariable() const override {
275 if (OuterRegionInfo)
276 return OuterRegionInfo->getThreadIDVariable();
277 return nullptr;
278 }
279
280 /// Get an LValue for the current ThreadID variable.
281 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
284 llvm_unreachable("No LValue for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No LValue for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 284)
;
285 }
286
287 /// Get the name of the capture helper.
288 StringRef getHelperName() const override {
289 if (auto *OuterRegionInfo = getOldCSI())
290 return OuterRegionInfo->getHelperName();
291 llvm_unreachable("No helper name for inlined OpenMP construct")::llvm::llvm_unreachable_internal("No helper name for inlined OpenMP construct"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 291)
;
292 }
293
294 void emitUntiedSwitch(CodeGenFunction &CGF) override {
295 if (OuterRegionInfo)
296 OuterRegionInfo->emitUntiedSwitch(CGF);
297 }
298
299 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
300
301 static bool classof(const CGCapturedStmtInfo *Info) {
302 return CGOpenMPRegionInfo::classof(Info) &&
303 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
304 }
305
306 ~CGOpenMPInlinedRegionInfo() override = default;
307
308private:
309 /// CodeGen info about outer OpenMP region.
310 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
311 CGOpenMPRegionInfo *OuterRegionInfo;
312};
313
314/// API for captured statement code generation in OpenMP target
315/// constructs. For this captures, implicit parameters are used instead of the
316/// captured fields. The name of the target region has to be unique in a given
317/// application so it is provided by the client, because only the client has
318/// the information to generate that.
319class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
320public:
321 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
322 const RegionCodeGenTy &CodeGen, StringRef HelperName)
323 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
324 /*HasCancel=*/false),
325 HelperName(HelperName) {}
326
327 /// This is unused for target regions because each starts executing
328 /// with a single thread.
329 const VarDecl *getThreadIDVariable() const override { return nullptr; }
330
331 /// Get the name of the capture helper.
332 StringRef getHelperName() const override { return HelperName; }
333
334 static bool classof(const CGCapturedStmtInfo *Info) {
335 return CGOpenMPRegionInfo::classof(Info) &&
336 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
337 }
338
339private:
340 StringRef HelperName;
341};
342
343static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
344 llvm_unreachable("No codegen for expressions")::llvm::llvm_unreachable_internal("No codegen for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 344)
;
345}
346/// API for generation of expressions captured in a innermost OpenMP
347/// region.
348class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
349public:
350 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
351 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
352 OMPD_unknown,
353 /*HasCancel=*/false),
354 PrivScope(CGF) {
355 // Make sure the globals captured in the provided statement are local by
356 // using the privatization logic. We assume the same variable is not
357 // captured more than once.
358 for (const auto &C : CS.captures()) {
359 if (!C.capturesVariable() && !C.capturesVariableByCopy())
360 continue;
361
362 const VarDecl *VD = C.getCapturedVar();
363 if (VD->isLocalVarDeclOrParm())
364 continue;
365
366 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
367 /*RefersToEnclosingVariableOrCapture=*/false,
368 VD->getType().getNonReferenceType(), VK_LValue,
369 C.getLocation());
370 PrivScope.addPrivate(
371 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
372 }
373 (void)PrivScope.Privatize();
374 }
375
376 /// Lookup the captured field decl for a variable.
377 const FieldDecl *lookup(const VarDecl *VD) const override {
378 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
379 return FD;
380 return nullptr;
381 }
382
383 /// Emit the captured statement body.
384 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
385 llvm_unreachable("No body for expressions")::llvm::llvm_unreachable_internal("No body for expressions", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 385)
;
386 }
387
388 /// Get a variable or parameter for storing global thread id
389 /// inside OpenMP construct.
390 const VarDecl *getThreadIDVariable() const override {
391 llvm_unreachable("No thread id for expressions")::llvm::llvm_unreachable_internal("No thread id for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 391)
;
392 }
393
394 /// Get the name of the capture helper.
395 StringRef getHelperName() const override {
396 llvm_unreachable("No helper name for expressions")::llvm::llvm_unreachable_internal("No helper name for expressions"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 396)
;
397 }
398
399 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
400
401private:
402 /// Private scope to capture global variables.
403 CodeGenFunction::OMPPrivateScope PrivScope;
404};
405
406/// RAII for emitting code of OpenMP constructs.
407class InlinedOpenMPRegionRAII {
408 CodeGenFunction &CGF;
409 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
410 FieldDecl *LambdaThisCaptureField = nullptr;
411 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
412
413public:
414 /// Constructs region for combined constructs.
415 /// \param CodeGen Code generation sequence for combined directives. Includes
416 /// a list of functions used for code generation of implicitly inlined
417 /// regions.
418 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
419 OpenMPDirectiveKind Kind, bool HasCancel)
420 : CGF(CGF) {
421 // Start emission for the construct.
422 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
423 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
424 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
425 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
426 CGF.LambdaThisCaptureField = nullptr;
427 BlockInfo = CGF.BlockInfo;
428 CGF.BlockInfo = nullptr;
429 }
430
431 ~InlinedOpenMPRegionRAII() {
432 // Restore original CapturedStmtInfo only if we're done with code emission.
433 auto *OldCSI =
434 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
435 delete CGF.CapturedStmtInfo;
436 CGF.CapturedStmtInfo = OldCSI;
437 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
438 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
439 CGF.BlockInfo = BlockInfo;
440 }
441};
442
443/// Values for bit flags used in the ident_t to describe the fields.
444/// All enumeric elements are named and described in accordance with the code
445/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
446enum OpenMPLocationFlags : unsigned {
447 /// Use trampoline for internal microtask.
448 OMP_IDENT_IMD = 0x01,
449 /// Use c-style ident structure.
450 OMP_IDENT_KMPC = 0x02,
451 /// Atomic reduction option for kmpc_reduce.
452 OMP_ATOMIC_REDUCE = 0x10,
453 /// Explicit 'barrier' directive.
454 OMP_IDENT_BARRIER_EXPL = 0x20,
455 /// Implicit barrier in code.
456 OMP_IDENT_BARRIER_IMPL = 0x40,
457 /// Implicit barrier in 'for' directive.
458 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
459 /// Implicit barrier in 'sections' directive.
460 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
461 /// Implicit barrier in 'single' directive.
462 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
463 /// Call of __kmp_for_static_init for static loop.
464 OMP_IDENT_WORK_LOOP = 0x200,
465 /// Call of __kmp_for_static_init for sections.
466 OMP_IDENT_WORK_SECTIONS = 0x400,
467 /// Call of __kmp_for_static_init for distribute.
468 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
469 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_IDENT_WORK_DISTRIBUTE
470};
471
472namespace {
473LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()using ::llvm::BitmaskEnumDetail::operator~; using ::llvm::BitmaskEnumDetail
::operator|; using ::llvm::BitmaskEnumDetail::operator&; using
::llvm::BitmaskEnumDetail::operator^; using ::llvm::BitmaskEnumDetail
::operator|=; using ::llvm::BitmaskEnumDetail::operator&=
; using ::llvm::BitmaskEnumDetail::operator^=
;
474/// Values for bit flags for marking which requires clauses have been used.
475enum OpenMPOffloadingRequiresDirFlags : int64_t {
476 /// flag undefined.
477 OMP_REQ_UNDEFINED = 0x000,
478 /// no requires clause present.
479 OMP_REQ_NONE = 0x001,
480 /// reverse_offload clause.
481 OMP_REQ_REVERSE_OFFLOAD = 0x002,
482 /// unified_address clause.
483 OMP_REQ_UNIFIED_ADDRESS = 0x004,
484 /// unified_shared_memory clause.
485 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
486 /// dynamic_allocators clause.
487 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
488 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)LLVM_BITMASK_LARGEST_ENUMERATOR = OMP_REQ_DYNAMIC_ALLOCATORS
489};
490
491enum OpenMPOffloadingReservedDeviceIDs {
492 /// Device ID if the device was not defined, runtime should get it
493 /// from environment variables in the spec.
494 OMP_DEVICEID_UNDEF = -1,
495};
496} // anonymous namespace
497
498/// Describes ident structure that describes a source location.
499/// All descriptions are taken from
500/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
501/// Original structure:
502/// typedef struct ident {
503/// kmp_int32 reserved_1; /**< might be used in Fortran;
504/// see above */
505/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
506/// KMP_IDENT_KMPC identifies this union
507/// member */
508/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
509/// see above */
510///#if USE_ITT_BUILD
511/// /* but currently used for storing
512/// region-specific ITT */
513/// /* contextual information. */
514///#endif /* USE_ITT_BUILD */
515/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
516/// C++ */
517/// char const *psource; /**< String describing the source location.
518/// The string is composed of semi-colon separated
519// fields which describe the source file,
520/// the function and a pair of line numbers that
521/// delimit the construct.
522/// */
523/// } ident_t;
524enum IdentFieldIndex {
525 /// might be used in Fortran
526 IdentField_Reserved_1,
527 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
528 IdentField_Flags,
529 /// Not really used in Fortran any more
530 IdentField_Reserved_2,
531 /// Source[4] in Fortran, do not use for C++
532 IdentField_Reserved_3,
533 /// String describing the source location. The string is composed of
534 /// semi-colon separated fields which describe the source file, the function
535 /// and a pair of line numbers that delimit the construct.
536 IdentField_PSource
537};
538
539/// Schedule types for 'omp for' loops (these enumerators are taken from
540/// the enum sched_type in kmp.h).
541enum OpenMPSchedType {
542 /// Lower bound for default (unordered) versions.
543 OMP_sch_lower = 32,
544 OMP_sch_static_chunked = 33,
545 OMP_sch_static = 34,
546 OMP_sch_dynamic_chunked = 35,
547 OMP_sch_guided_chunked = 36,
548 OMP_sch_runtime = 37,
549 OMP_sch_auto = 38,
550 /// static with chunk adjustment (e.g., simd)
551 OMP_sch_static_balanced_chunked = 45,
552 /// Lower bound for 'ordered' versions.
553 OMP_ord_lower = 64,
554 OMP_ord_static_chunked = 65,
555 OMP_ord_static = 66,
556 OMP_ord_dynamic_chunked = 67,
557 OMP_ord_guided_chunked = 68,
558 OMP_ord_runtime = 69,
559 OMP_ord_auto = 70,
560 OMP_sch_default = OMP_sch_static,
561 /// dist_schedule types
562 OMP_dist_sch_static_chunked = 91,
563 OMP_dist_sch_static = 92,
564 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
565 /// Set if the monotonic schedule modifier was present.
566 OMP_sch_modifier_monotonic = (1 << 29),
567 /// Set if the nonmonotonic schedule modifier was present.
568 OMP_sch_modifier_nonmonotonic = (1 << 30),
569};
570
571/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
572/// region.
573class CleanupTy final : public EHScopeStack::Cleanup {
574 PrePostActionTy *Action;
575
576public:
577 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
578 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
579 if (!CGF.HaveInsertPoint())
580 return;
581 Action->Exit(CGF);
582 }
583};
584
585} // anonymous namespace
586
587void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
588 CodeGenFunction::RunCleanupsScope Scope(CGF);
589 if (PrePostAction) {
590 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
591 Callback(CodeGen, CGF, *PrePostAction);
592 } else {
593 PrePostActionTy Action;
594 Callback(CodeGen, CGF, Action);
595 }
596}
597
598/// Check if the combiner is a call to UDR combiner and if it is so return the
599/// UDR decl used for reduction.
600static const OMPDeclareReductionDecl *
601getReductionInit(const Expr *ReductionOp) {
602 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
603 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
604 if (const auto *DRE =
605 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
606 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
607 return DRD;
608 return nullptr;
609}
610
611static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
612 const OMPDeclareReductionDecl *DRD,
613 const Expr *InitOp,
614 Address Private, Address Original,
615 QualType Ty) {
616 if (DRD->getInitializer()) {
617 std::pair<llvm::Function *, llvm::Function *> Reduction =
618 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
619 const auto *CE = cast<CallExpr>(InitOp);
620 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
621 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
622 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
623 const auto *LHSDRE =
624 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
625 const auto *RHSDRE =
626 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
627 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
628 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
629 [=]() { return Private; });
630 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
631 [=]() { return Original; });
632 (void)PrivateScope.Privatize();
633 RValue Func = RValue::get(Reduction.second);
634 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
635 CGF.EmitIgnoredExpr(InitOp);
636 } else {
637 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
638 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
639 auto *GV = new llvm::GlobalVariable(
640 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
641 llvm::GlobalValue::PrivateLinkage, Init, Name);
642 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
643 RValue InitRVal;
644 switch (CGF.getEvaluationKind(Ty)) {
645 case TEK_Scalar:
646 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
647 break;
648 case TEK_Complex:
649 InitRVal =
650 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
651 break;
652 case TEK_Aggregate:
653 InitRVal = RValue::getAggregate(LV.getAddress(CGF));
654 break;
655 }
656 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
657 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
658 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
659 /*IsInitializer=*/false);
660 }
661}
662
663/// Emit initialization of arrays of complex types.
664/// \param DestAddr Address of the array.
665/// \param Type Type of array.
666/// \param Init Initial expression of array.
667/// \param SrcAddr Address of the original array.
668static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
669 QualType Type, bool EmitDeclareReductionInit,
670 const Expr *Init,
671 const OMPDeclareReductionDecl *DRD,
672 Address SrcAddr = Address::invalid()) {
673 // Perform element-by-element initialization.
674 QualType ElementTy;
675
676 // Drill down to the base element type on both arrays.
677 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
678 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
679 DestAddr =
680 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
681 if (DRD)
682 SrcAddr =
683 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
684
685 llvm::Value *SrcBegin = nullptr;
686 if (DRD)
687 SrcBegin = SrcAddr.getPointer();
688 llvm::Value *DestBegin = DestAddr.getPointer();
689 // Cast from pointer to array type to pointer to single element.
690 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
691 // The basic structure here is a while-do loop.
692 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
693 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
694 llvm::Value *IsEmpty =
695 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
696 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
697
698 // Enter the loop body, making that address the current address.
699 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
700 CGF.EmitBlock(BodyBB);
701
702 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
703
704 llvm::PHINode *SrcElementPHI = nullptr;
705 Address SrcElementCurrent = Address::invalid();
706 if (DRD) {
707 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
708 "omp.arraycpy.srcElementPast");
709 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
710 SrcElementCurrent =
711 Address(SrcElementPHI,
712 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
713 }
714 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
715 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
716 DestElementPHI->addIncoming(DestBegin, EntryBB);
717 Address DestElementCurrent =
718 Address(DestElementPHI,
719 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
720
721 // Emit copy.
722 {
723 CodeGenFunction::RunCleanupsScope InitScope(CGF);
724 if (EmitDeclareReductionInit) {
725 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
726 SrcElementCurrent, ElementTy);
727 } else
728 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
729 /*IsInitializer=*/false);
730 }
731
732 if (DRD) {
733 // Shift the address forward by one element.
734 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
735 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
736 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
737 }
738
739 // Shift the address forward by one element.
740 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
741 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
742 // Check whether we've reached the end.
743 llvm::Value *Done =
744 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
745 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
746 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
747
748 // Done.
749 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
750}
751
752LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
753 return CGF.EmitOMPSharedLValue(E);
754}
755
756LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
757 const Expr *E) {
758 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
759 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
760 return LValue();
761}
762
763void ReductionCodeGen::emitAggregateInitialization(
764 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
765 const OMPDeclareReductionDecl *DRD) {
766 // Emit VarDecl with copy init for arrays.
767 // Get the address of the original variable captured in current
768 // captured region.
769 const auto *PrivateVD =
770 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
771 bool EmitDeclareReductionInit =
772 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
773 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
774 EmitDeclareReductionInit,
775 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
776 : PrivateVD->getInit(),
777 DRD, SharedLVal.getAddress(CGF));
778}
779
780ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
781 ArrayRef<const Expr *> Origs,
782 ArrayRef<const Expr *> Privates,
783 ArrayRef<const Expr *> ReductionOps) {
784 ClausesData.reserve(Shareds.size());
785 SharedAddresses.reserve(Shareds.size());
786 Sizes.reserve(Shareds.size());
787 BaseDecls.reserve(Shareds.size());
788 const auto *IOrig = Origs.begin();
789 const auto *IPriv = Privates.begin();
790 const auto *IRed = ReductionOps.begin();
791 for (const Expr *Ref : Shareds) {
792 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
793 std::advance(IOrig, 1);
794 std::advance(IPriv, 1);
795 std::advance(IRed, 1);
796 }
797}
798
799void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
800 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
801 "Number of generated lvalues must be exactly N.")((SharedAddresses.size() == N && OrigAddresses.size()
== N && "Number of generated lvalues must be exactly N."
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() == N && OrigAddresses.size() == N && \"Number of generated lvalues must be exactly N.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 801, __PRETTY_FUNCTION__))
;
802 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
803 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
804 SharedAddresses.emplace_back(First, Second);
805 if (ClausesData[N].Shared == ClausesData[N].Ref) {
806 OrigAddresses.emplace_back(First, Second);
807 } else {
808 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
809 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
810 OrigAddresses.emplace_back(First, Second);
811 }
812}
813
814void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
815 const auto *PrivateVD =
816 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
817 QualType PrivateType = PrivateVD->getType();
818 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
819 if (!PrivateType->isVariablyModifiedType()) {
820 Sizes.emplace_back(
821 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
822 nullptr);
823 return;
824 }
825 llvm::Value *Size;
826 llvm::Value *SizeInChars;
827 auto *ElemType =
828 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType())
829 ->getElementType();
830 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
831 if (AsArraySection) {
832 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
833 OrigAddresses[N].first.getPointer(CGF));
834 Size = CGF.Builder.CreateNUWAdd(
835 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
836 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
837 } else {
838 SizeInChars =
839 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
840 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
841 }
842 Sizes.emplace_back(SizeInChars, Size);
843 CodeGenFunction::OpaqueValueMapping OpaqueMap(
844 CGF,
845 cast<OpaqueValueExpr>(
846 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
847 RValue::get(Size));
848 CGF.EmitVariablyModifiedType(PrivateType);
849}
850
851void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
852 llvm::Value *Size) {
853 const auto *PrivateVD =
854 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
855 QualType PrivateType = PrivateVD->getType();
856 if (!PrivateType->isVariablyModifiedType()) {
857 assert(!Size && !Sizes[N].second &&((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
858 "Size should be nullptr for non-variably modified reduction "((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
859 "items.")((!Size && !Sizes[N].second && "Size should be nullptr for non-variably modified reduction "
"items.") ? static_cast<void> (0) : __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 859, __PRETTY_FUNCTION__))
;
860 return;
861 }
862 CodeGenFunction::OpaqueValueMapping OpaqueMap(
863 CGF,
864 cast<OpaqueValueExpr>(
865 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
866 RValue::get(Size));
867 CGF.EmitVariablyModifiedType(PrivateType);
868}
869
870void ReductionCodeGen::emitInitialization(
871 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
872 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
873 assert(SharedAddresses.size() > N && "No variable was generated")((SharedAddresses.size() > N && "No variable was generated"
) ? static_cast<void> (0) : __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 873, __PRETTY_FUNCTION__))
;
874 const auto *PrivateVD =
875 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
876 const OMPDeclareReductionDecl *DRD =
877 getReductionInit(ClausesData[N].ReductionOp);
878 QualType PrivateType = PrivateVD->getType();
879 PrivateAddr = CGF.Builder.CreateElementBitCast(
880 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
881 QualType SharedType = SharedAddresses[N].first.getType();
882 SharedLVal = CGF.MakeAddrLValue(
883 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
884 CGF.ConvertTypeForMem(SharedType)),
885 SharedType, SharedAddresses[N].first.getBaseInfo(),
886 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
887 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
888 if (DRD && DRD->getInitializer())
889 (void)DefaultInit(CGF);
890 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
891 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
892 (void)DefaultInit(CGF);
893 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
894 PrivateAddr, SharedLVal.getAddress(CGF),
895 SharedLVal.getType());
896 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
897 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
898 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
899 PrivateVD->getType().getQualifiers(),
900 /*IsInitializer=*/false);
901 }
902}
903
904bool ReductionCodeGen::needCleanups(unsigned N) {
905 const auto *PrivateVD =
906 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
907 QualType PrivateType = PrivateVD->getType();
908 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
909 return DTorKind != QualType::DK_none;
910}
911
912void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
913 Address PrivateAddr) {
914 const auto *PrivateVD =
915 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
916 QualType PrivateType = PrivateVD->getType();
917 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
918 if (needCleanups(N)) {
919 PrivateAddr = CGF.Builder.CreateElementBitCast(
920 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
921 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
922 }
923}
924
925static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
926 LValue BaseLV) {
927 BaseTy = BaseTy.getNonReferenceType();
928 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
929 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
930 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
931 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
932 } else {
933 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
934 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
935 }
936 BaseTy = BaseTy->getPointeeType();
937 }
938 return CGF.MakeAddrLValue(
939 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
940 CGF.ConvertTypeForMem(ElTy)),
941 BaseLV.getType(), BaseLV.getBaseInfo(),
942 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
943}
944
945static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
946 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
947 llvm::Value *Addr) {
948 Address Tmp = Address::invalid();
949 Address TopTmp = Address::invalid();
950 Address MostTopTmp = Address::invalid();
951 BaseTy = BaseTy.getNonReferenceType();
952 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
953 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
954 Tmp = CGF.CreateMemTemp(BaseTy);
955 if (TopTmp.isValid())
956 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
957 else
958 MostTopTmp = Tmp;
959 TopTmp = Tmp;
960 BaseTy = BaseTy->getPointeeType();
961 }
962 llvm::Type *Ty = BaseLVType;
963 if (Tmp.isValid())
964 Ty = Tmp.getElementType();
965 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
966 if (Tmp.isValid()) {
967 CGF.Builder.CreateStore(Addr, Tmp);
968 return MostTopTmp;
969 }
970 return Address(Addr, BaseLVAlignment);
971}
972
973static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
974 const VarDecl *OrigVD = nullptr;
975 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
976 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
977 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
978 Base = TempOASE->getBase()->IgnoreParenImpCasts();
979 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
980 Base = TempASE->getBase()->IgnoreParenImpCasts();
981 DE = cast<DeclRefExpr>(Base);
982 OrigVD = cast<VarDecl>(DE->getDecl());
983 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
984 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
985 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
986 Base = TempASE->getBase()->IgnoreParenImpCasts();
987 DE = cast<DeclRefExpr>(Base);
988 OrigVD = cast<VarDecl>(DE->getDecl());
989 }
990 return OrigVD;
991}
992
993Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
994 Address PrivateAddr) {
995 const DeclRefExpr *DE;
996 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
997 BaseDecls.emplace_back(OrigVD);
998 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
999 LValue BaseLValue =
1000 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1001 OriginalBaseLValue);
1002 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1003 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1004 llvm::Value *PrivatePointer =
1005 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1006 PrivateAddr.getPointer(),
1007 SharedAddresses[N].first.getAddress(CGF).getType());
1008 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1009 return castToBase(CGF, OrigVD->getType(),
1010 SharedAddresses[N].first.getType(),
1011 OriginalBaseLValue.getAddress(CGF).getType(),
1012 OriginalBaseLValue.getAlignment(), Ptr);
1013 }
1014 BaseDecls.emplace_back(
1015 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1016 return PrivateAddr;
1017}
1018
1019bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1020 const OMPDeclareReductionDecl *DRD =
1021 getReductionInit(ClausesData[N].ReductionOp);
1022 return DRD && DRD->getInitializer();
1023}
1024
1025LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1026 return CGF.EmitLoadOfPointerLValue(
1027 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1028 getThreadIDVariable()->getType()->castAs<PointerType>());
1029}
1030
1031void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1032 if (!CGF.HaveInsertPoint())
1033 return;
1034 // 1.2.2 OpenMP Language Terminology
1035 // Structured block - An executable statement with a single entry at the
1036 // top and a single exit at the bottom.
1037 // The point of exit cannot be a branch out of the structured block.
1038 // longjmp() and throw() must not violate the entry/exit criteria.
1039 CGF.EHStack.pushTerminate();
1040 CodeGen(CGF);
1041 CGF.EHStack.popTerminate();
1042}
1043
1044LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1045 CodeGenFunction &CGF) {
1046 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1047 getThreadIDVariable()->getType(),
1048 AlignmentSource::Decl);
1049}
1050
1051static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1052 QualType FieldTy) {
1053 auto *Field = FieldDecl::Create(
1054 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1055 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1056 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1057 Field->setAccess(AS_public);
1058 DC->addDecl(Field);
1059 return Field;
1060}
1061
1062CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1063 StringRef Separator)
1064 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1065 OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
1066 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1067
1068 // Initialize Types used in OpenMPIRBuilder from OMPKinds.def
1069 OMPBuilder.initialize();
1070 loadOffloadInfoMetadata();
1071}
1072
1073void CGOpenMPRuntime::clear() {
1074 InternalVars.clear();
1075 // Clean non-target variable declarations possibly used only in debug info.
1076 for (const auto &Data : EmittedNonTargetVariables) {
1077 if (!Data.getValue().pointsToAliveValue())
1078 continue;
1079 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1080 if (!GV)
1081 continue;
1082 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1083 continue;
1084 GV->eraseFromParent();
1085 }
1086}
1087
1088std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1089 SmallString<128> Buffer;
1090 llvm::raw_svector_ostream OS(Buffer);
1091 StringRef Sep = FirstSeparator;
1092 for (StringRef Part : Parts) {
1093 OS << Sep << Part;
1094 Sep = Separator;
1095 }
1096 return std::string(OS.str());
1097}
1098
1099static llvm::Function *
1100emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1101 const Expr *CombinerInitializer, const VarDecl *In,
1102 const VarDecl *Out, bool IsCombiner) {
1103 // void .omp_combiner.(Ty *in, Ty *out);
1104 ASTContext &C = CGM.getContext();
1105 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1106 FunctionArgList Args;
1107 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1108 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1109 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1110 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1111 Args.push_back(&OmpOutParm);
1112 Args.push_back(&OmpInParm);
1113 const CGFunctionInfo &FnInfo =
1114 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1115 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1116 std::string Name = CGM.getOpenMPRuntime().getName(
1117 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1118 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1119 Name, &CGM.getModule());
1120 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1121 if (CGM.getLangOpts().Optimize) {
1122 Fn->removeFnAttr(llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1125 }
1126 CodeGenFunction CGF(CGM);
1127 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1128 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1129 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1130 Out->getLocation());
1131 CodeGenFunction::OMPPrivateScope Scope(CGF);
1132 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1133 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1134 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1135 .getAddress(CGF);
1136 });
1137 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1138 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1139 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1140 .getAddress(CGF);
1141 });
1142 (void)Scope.Privatize();
1143 if (!IsCombiner && Out->hasInit() &&
1144 !CGF.isTrivialInitializer(Out->getInit())) {
1145 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1146 Out->getType().getQualifiers(),
1147 /*IsInitializer=*/true);
1148 }
1149 if (CombinerInitializer)
1150 CGF.EmitIgnoredExpr(CombinerInitializer);
1151 Scope.ForceCleanup();
1152 CGF.FinishFunction();
1153 return Fn;
1154}
1155
1156void CGOpenMPRuntime::emitUserDefinedReduction(
1157 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1158 if (UDRMap.count(D) > 0)
1159 return;
1160 llvm::Function *Combiner = emitCombinerOrInitializer(
1161 CGM, D->getType(), D->getCombiner(),
1162 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1163 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1164 /*IsCombiner=*/true);
1165 llvm::Function *Initializer = nullptr;
1166 if (const Expr *Init = D->getInitializer()) {
1167 Initializer = emitCombinerOrInitializer(
1168 CGM, D->getType(),
1169 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1170 : nullptr,
1171 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1172 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1173 /*IsCombiner=*/false);
1174 }
1175 UDRMap.try_emplace(D, Combiner, Initializer);
1176 if (CGF) {
1177 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1178 Decls.second.push_back(D);
1179 }
1180}
1181
1182std::pair<llvm::Function *, llvm::Function *>
1183CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1184 auto I = UDRMap.find(D);
1185 if (I != UDRMap.end())
1186 return I->second;
1187 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1188 return UDRMap.lookup(D);
1189}
1190
1191namespace {
1192// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1193// Builder if one is present.
1194struct PushAndPopStackRAII {
1195 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1196 bool HasCancel)
1197 : OMPBuilder(OMPBuilder) {
1198 if (!OMPBuilder)
1199 return;
1200
1201 // The following callback is the crucial part of clangs cleanup process.
1202 //
1203 // NOTE:
1204 // Once the OpenMPIRBuilder is used to create parallel regions (and
1205 // similar), the cancellation destination (Dest below) is determined via
1206 // IP. That means if we have variables to finalize we split the block at IP,
1207 // use the new block (=BB) as destination to build a JumpDest (via
1208 // getJumpDestInCurrentScope(BB)) which then is fed to
1209 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1210 // to push & pop an FinalizationInfo object.
1211 // The FiniCB will still be needed but at the point where the
1212 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1213 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1214 assert(IP.getBlock()->end() == IP.getPoint() &&((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
1215 "Clang CG should cause non-terminated block!")((IP.getBlock()->end() == IP.getPoint() && "Clang CG should cause non-terminated block!"
) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() == IP.getPoint() && \"Clang CG should cause non-terminated block!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1215, __PRETTY_FUNCTION__))
;
1216 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1217 CGF.Builder.restoreIP(IP);
1218 CodeGenFunction::JumpDest Dest =
1219 CGF.getOMPCancelDestination(OMPD_parallel);
1220 CGF.EmitBranchThroughCleanup(Dest);
1221 };
1222
1223 // TODO: Remove this once we emit parallel regions through the
1224 // OpenMPIRBuilder as it can do this setup internally.
1225 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1226 {FiniCB, OMPD_parallel, HasCancel});
1227 OMPBuilder->pushFinalizationCB(std::move(FI));
1228 }
1229 ~PushAndPopStackRAII() {
1230 if (OMPBuilder)
1231 OMPBuilder->popFinalizationCB();
1232 }
1233 llvm::OpenMPIRBuilder *OMPBuilder;
1234};
1235} // namespace
1236
1237static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1238 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1239 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1240 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1241 assert(ThreadIDVar->getType()->isPointerType() &&((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
1242 "thread id variable must be of type kmp_int32 *")((ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 *"
) ? static_cast<void> (0) : __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1242, __PRETTY_FUNCTION__))
;
1243 CodeGenFunction CGF(CGM, true);
1244 bool HasCancel = false;
1245 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1246 HasCancel = OPD->hasCancel();
1247 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1248 HasCancel = OPD->hasCancel();
1249 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1250 HasCancel = OPSD->hasCancel();
1251 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1254 HasCancel = OPFD->hasCancel();
1255 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1256 HasCancel = OPFD->hasCancel();
1257 else if (const auto *OPFD =
1258 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1259 HasCancel = OPFD->hasCancel();
1260 else if (const auto *OPFD =
1261 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1262 HasCancel = OPFD->hasCancel();
1263
1264 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1265 // parallel region to make cancellation barriers work properly.
1266 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1267 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel);
1268 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1269 HasCancel, OutlinedHelperName);
1270 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1271 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1272}
1273
1274llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1275 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1276 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1277 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1278 return emitParallelOrTeamsOutlinedFunction(
1279 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1280}
1281
1282llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1283 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1284 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1285 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1286 return emitParallelOrTeamsOutlinedFunction(
1287 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1288}
1289
1290llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1291 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1292 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1293 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1294 bool Tied, unsigned &NumberOfParts) {
1295 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1296 PrePostActionTy &) {
1297 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1298 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1299 llvm::Value *TaskArgs[] = {
1300 UpLoc, ThreadID,
1301 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1302 TaskTVar->getType()->castAs<PointerType>())
1303 .getPointer(CGF)};
1304 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1305 CGM.getModule(), OMPRTL___kmpc_omp_task),
1306 TaskArgs);
1307 };
1308 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1309 UntiedCodeGen);
1310 CodeGen.setAction(Action);
1311 assert(!ThreadIDVar->getType()->isPointerType() &&((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
1312 "thread id variable must be of type kmp_int32 for tasks")((!ThreadIDVar->getType()->isPointerType() && "thread id variable must be of type kmp_int32 for tasks"
) ? static_cast<void> (0) : __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1312, __PRETTY_FUNCTION__))
;
1313 const OpenMPDirectiveKind Region =
1314 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1315 : OMPD_task;
1316 const CapturedStmt *CS = D.getCapturedStmt(Region);
1317 bool HasCancel = false;
1318 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1319 HasCancel = TD->hasCancel();
1320 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1321 HasCancel = TD->hasCancel();
1322 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1323 HasCancel = TD->hasCancel();
1324 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1325 HasCancel = TD->hasCancel();
1326
1327 CodeGenFunction CGF(CGM, true);
1328 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1329 InnermostKind, HasCancel, Action);
1330 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1331 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1332 if (!Tied)
1333 NumberOfParts = Action.getNumberOfParts();
1334 return Res;
1335}
1336
1337static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1338 const RecordDecl *RD, const CGRecordLayout &RL,
1339 ArrayRef<llvm::Constant *> Data) {
1340 llvm::StructType *StructTy = RL.getLLVMType();
1341 unsigned PrevIdx = 0;
1342 ConstantInitBuilder CIBuilder(CGM);
1343 auto DI = Data.begin();
1344 for (const FieldDecl *FD : RD->fields()) {
1345 unsigned Idx = RL.getLLVMFieldNo(FD);
1346 // Fill the alignment.
1347 for (unsigned I = PrevIdx; I < Idx; ++I)
1348 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1349 PrevIdx = Idx + 1;
1350 Fields.add(*DI);
1351 ++DI;
1352 }
1353}
1354
1355template <class... As>
1356static llvm::GlobalVariable *
1357createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1358 ArrayRef<llvm::Constant *> Data, const Twine &Name,
1359 As &&... Args) {
1360 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1361 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1362 ConstantInitBuilder CIBuilder(CGM);
1363 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1364 buildStructValue(Fields, CGM, RD, RL, Data);
1365 return Fields.finishAndCreateGlobal(
1366 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1367 std::forward<As>(Args)...);
1368}
1369
1370template <typename T>
1371static void
1372createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1373 ArrayRef<llvm::Constant *> Data,
1374 T &Parent) {
1375 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1376 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1377 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1378 buildStructValue(Fields, CGM, RD, RL, Data);
1379 Fields.finishAndAddTo(Parent);
1380}
1381
1382void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1383 bool AtCurrentPoint) {
1384 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1385 assert(!Elem.second.ServiceInsertPt && "Insert point is set already.")((!Elem.second.ServiceInsertPt && "Insert point is set already."
) ? static_cast<void> (0) : __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1385, __PRETTY_FUNCTION__))
;
1386
1387 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1388 if (AtCurrentPoint) {
1389 Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1390 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1391 } else {
1392 Elem.second.ServiceInsertPt =
1393 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1394 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1395 }
1396}
1397
1398void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1399 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1400 if (Elem.second.ServiceInsertPt) {
1401 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1402 Elem.second.ServiceInsertPt = nullptr;
1403 Ptr->eraseFromParent();
1404 }
1405}
1406
1407static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1408 SourceLocation Loc,
1409 SmallString<128> &Buffer) {
1410 llvm::raw_svector_ostream OS(Buffer);
1411 // Build debug location
1412 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1413 OS << ";" << PLoc.getFilename() << ";";
1414 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1415 OS << FD->getQualifiedNameAsString();
1416 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1417 return OS.str();
1418}
1419
1420llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1421 SourceLocation Loc,
1422 unsigned Flags) {
1423 llvm::Constant *SrcLocStr;
1424 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1425 Loc.isInvalid()) {
1426 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr();
1427 } else {
1428 std::string FunctionName = "";
1429 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1430 FunctionName = FD->getQualifiedNameAsString();
1431 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1432 const char *FileName = PLoc.getFilename();
1433 unsigned Line = PLoc.getLine();
1434 unsigned Column = PLoc.getColumn();
1435 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName.c_str(), FileName,
1436 Line, Column);
1437 }
1438 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1439 return OMPBuilder.getOrCreateIdent(SrcLocStr, llvm::omp::IdentFlag(Flags),
1440 Reserved2Flags);
1441}
1442
1443llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1444 SourceLocation Loc) {
1445 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1445, __PRETTY_FUNCTION__))
;
1446 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1447 // the clang invariants used below might be broken.
1448 if (CGM.getLangOpts().OpenMPIRBuilder) {
1449 SmallString<128> Buffer;
1450 OMPBuilder.updateToLocation(CGF.Builder.saveIP());
1451 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1452 getIdentStringFromSourceLocation(CGF, Loc, Buffer));
1453 return OMPBuilder.getOrCreateThreadID(
1454 OMPBuilder.getOrCreateIdent(SrcLocStr));
1455 }
1456
1457 llvm::Value *ThreadID = nullptr;
1458 // Check whether we've already cached a load of the thread id in this
1459 // function.
1460 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1461 if (I != OpenMPLocThreadIDMap.end()) {
1462 ThreadID = I->second.ThreadID;
1463 if (ThreadID != nullptr)
1464 return ThreadID;
1465 }
1466 // If exceptions are enabled, do not use parameter to avoid possible crash.
1467 if (auto *OMPRegionInfo =
1468 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1469 if (OMPRegionInfo->getThreadIDVariable()) {
1470 // Check if this an outlined function with thread id passed as argument.
1471 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1472 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1473 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1474 !CGF.getLangOpts().CXXExceptions ||
1475 CGF.Builder.GetInsertBlock() == TopBlock ||
1476 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1477 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1478 TopBlock ||
1479 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1480 CGF.Builder.GetInsertBlock()) {
1481 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1482 // If value loaded in entry block, cache it and use it everywhere in
1483 // function.
1484 if (CGF.Builder.GetInsertBlock() == TopBlock) {
1485 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1486 Elem.second.ThreadID = ThreadID;
1487 }
1488 return ThreadID;
1489 }
1490 }
1491 }
1492
1493 // This is not an outlined function region - need to call __kmpc_int32
1494 // kmpc_global_thread_num(ident_t *loc).
1495 // Generate thread id value and cache this value for use across the
1496 // function.
1497 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1498 if (!Elem.second.ServiceInsertPt)
1499 setLocThreadIdInsertPt(CGF);
1500 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1501 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1502 llvm::CallInst *Call = CGF.Builder.CreateCall(
1503 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1504 OMPRTL___kmpc_global_thread_num),
1505 emitUpdateLocation(CGF, Loc));
1506 Call->setCallingConv(CGF.getRuntimeCC());
1507 Elem.second.ThreadID = Call;
1508 return Call;
1509}
1510
1511void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1512 assert(CGF.CurFn && "No function in current CodeGenFunction.")((CGF.CurFn && "No function in current CodeGenFunction."
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1512, __PRETTY_FUNCTION__))
;
1513 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1514 clearLocThreadIdInsertPt(CGF);
1515 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1516 }
1517 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1518 for(const auto *D : FunctionUDRMap[CGF.CurFn])
1519 UDRMap.erase(D);
1520 FunctionUDRMap.erase(CGF.CurFn);
1521 }
1522 auto I = FunctionUDMMap.find(CGF.CurFn);
1523 if (I != FunctionUDMMap.end()) {
1524 for(const auto *D : I->second)
1525 UDMMap.erase(D);
1526 FunctionUDMMap.erase(I);
1527 }
1528 LastprivateConditionalToTypes.erase(CGF.CurFn);
1529 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);
1530}
1531
1532llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1533 return OMPBuilder.IdentPtr;
1534}
1535
1536llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1537 if (!Kmpc_MicroTy) {
1538 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1539 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1540 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1541 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1542 }
1543 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1544}
1545
1546llvm::FunctionCallee
1547CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
1548 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
1549 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1549, __PRETTY_FUNCTION__))
;
1550 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
1551 : "__kmpc_for_static_init_4u")
1552 : (IVSigned ? "__kmpc_for_static_init_8"
1553 : "__kmpc_for_static_init_8u");
1554 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1555 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1556 llvm::Type *TypeParams[] = {
1557 getIdentTyPointerTy(), // loc
1558 CGM.Int32Ty, // tid
1559 CGM.Int32Ty, // schedtype
1560 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1561 PtrTy, // p_lower
1562 PtrTy, // p_upper
1563 PtrTy, // p_stride
1564 ITy, // incr
1565 ITy // chunk
1566 };
1567 auto *FnTy =
1568 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1569 return CGM.CreateRuntimeFunction(FnTy, Name);
1570}
1571
1572llvm::FunctionCallee
1573CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
1574 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
1575 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1575, __PRETTY_FUNCTION__))
;
1576 StringRef Name =
1577 IVSize == 32
1578 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
1579 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
1580 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1581 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
1582 CGM.Int32Ty, // tid
1583 CGM.Int32Ty, // schedtype
1584 ITy, // lower
1585 ITy, // upper
1586 ITy, // stride
1587 ITy // chunk
1588 };
1589 auto *FnTy =
1590 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1591 return CGM.CreateRuntimeFunction(FnTy, Name);
1592}
1593
1594llvm::FunctionCallee
1595CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
1596 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
1597 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1597, __PRETTY_FUNCTION__))
;
1598 StringRef Name =
1599 IVSize == 32
1600 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
1601 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
1602 llvm::Type *TypeParams[] = {
1603 getIdentTyPointerTy(), // loc
1604 CGM.Int32Ty, // tid
1605 };
1606 auto *FnTy =
1607 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1608 return CGM.CreateRuntimeFunction(FnTy, Name);
1609}
1610
1611llvm::FunctionCallee
1612CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
1613 assert((IVSize == 32 || IVSize == 64) &&(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
1614 "IV size is not compatible with the omp runtime")(((IVSize == 32 || IVSize == 64) && "IV size is not compatible with the omp runtime"
) ? static_cast<void> (0) : __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1614, __PRETTY_FUNCTION__))
;
1615 StringRef Name =
1616 IVSize == 32
1617 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
1618 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
1619 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1620 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1621 llvm::Type *TypeParams[] = {
1622 getIdentTyPointerTy(), // loc
1623 CGM.Int32Ty, // tid
1624 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
1625 PtrTy, // p_lower
1626 PtrTy, // p_upper
1627 PtrTy // p_stride
1628 };
1629 auto *FnTy =
1630 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1631 return CGM.CreateRuntimeFunction(FnTy, Name);
1632}
1633
1634/// Obtain information that uniquely identifies a target entry. This
1635/// consists of the file and device IDs as well as line number associated with
1636/// the relevant entry source location.
1637static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
1638 unsigned &DeviceID, unsigned &FileID,
1639 unsigned &LineNum) {
1640 SourceManager &SM = C.getSourceManager();
1641
1642 // The loc should be always valid and have a file ID (the user cannot use
1643 // #pragma directives in macros)
1644
1645 assert(Loc.isValid() && "Source location is expected to be always valid.")((Loc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1645, __PRETTY_FUNCTION__))
;
1646
1647 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
1648 assert(PLoc.isValid() && "Source location is expected to be always valid.")((PLoc.isValid() && "Source location is expected to be always valid."
) ? static_cast<void> (0) : __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1648, __PRETTY_FUNCTION__))
;
1649
1650 llvm::sys::fs::UniqueID ID;
1651 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
1652 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1653 << PLoc.getFilename() << EC.message();
1654
1655 DeviceID = ID.getDevice();
1656 FileID = ID.getFile();
1657 LineNum = PLoc.getLine();
1658}
1659
1660Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1661 if (CGM.getLangOpts().OpenMPSimd)
1662 return Address::invalid();
1663 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1664 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1665 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1666 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1667 HasRequiresUnifiedSharedMemory))) {
1668 SmallString<64> PtrName;
1669 {
1670 llvm::raw_svector_ostream OS(PtrName);
1671 OS << CGM.getMangledName(GlobalDecl(VD));
1672 if (!VD->isExternallyVisible()) {
1673 unsigned DeviceID, FileID, Line;
1674 getTargetEntryUniqueInfo(CGM.getContext(),
1675 VD->getCanonicalDecl()->getBeginLoc(),
1676 DeviceID, FileID, Line);
1677 OS << llvm::format("_%x", FileID);
1678 }
1679 OS << "_decl_tgt_ref_ptr";
1680 }
1681 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
1682 if (!Ptr) {
1683 QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
1684 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
1685 PtrName);
1686
1687 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1688 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1689
1690 if (!CGM.getLangOpts().OpenMPIsDevice)
1691 GV->setInitializer(CGM.GetAddrOfGlobal(VD));
1692 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
1693 }
1694 return Address(Ptr, CGM.getContext().getDeclAlign(VD));
1695 }
1696 return Address::invalid();
1697}
1698
1699llvm::Constant *
1700CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1701 assert(!CGM.getLangOpts().OpenMPUseTLS ||((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
1702 !CGM.getContext().getTargetInfo().isTLSSupported())((!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo
().isTLSSupported()) ? static_cast<void> (0) : __assert_fail
("!CGM.getLangOpts().OpenMPUseTLS || !CGM.getContext().getTargetInfo().isTLSSupported()"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1702, __PRETTY_FUNCTION__))
;
1703 // Lookup the entry, lazily creating it if necessary.
1704 std::string Suffix = getName({"cache", ""});
1705 return getOrCreateInternalVariable(
1706 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
1707}
1708
1709Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1710 const VarDecl *VD,
1711 Address VDAddr,
1712 SourceLocation Loc) {
1713 if (CGM.getLangOpts().OpenMPUseTLS &&
1714 CGM.getContext().getTargetInfo().isTLSSupported())
1715 return VDAddr;
1716
1717 llvm::Type *VarTy = VDAddr.getElementType();
1718 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1719 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1720 CGM.Int8PtrTy),
1721 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1722 getOrCreateThreadPrivateCache(VD)};
1723 return Address(CGF.EmitRuntimeCall(
1724 OMPBuilder.getOrCreateRuntimeFunction(
1725 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1726 Args),
1727 VDAddr.getAlignment());
1728}
1729
1730void CGOpenMPRuntime::emitThreadPrivateVarInit(
1731 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1732 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1733 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1734 // library.
1735 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1736 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1737 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1738 OMPLoc);
1739 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1740 // to register constructor/destructor for variable.
1741 llvm::Value *Args[] = {
1742 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
1743 Ctor, CopyCtor, Dtor};
1744 CGF.EmitRuntimeCall(
1745 OMPBuilder.getOrCreateRuntimeFunction(
1746 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1747 Args);
1748}
1749
1750llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1751 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1752 bool PerformInit, CodeGenFunction *CGF) {
1753 if (CGM.getLangOpts().OpenMPUseTLS &&
1754 CGM.getContext().getTargetInfo().isTLSSupported())
1755 return nullptr;
1756
1757 VD = VD->getDefinition(CGM.getContext());
1758 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
1759 QualType ASTTy = VD->getType();
1760
1761 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1762 const Expr *Init = VD->getAnyInitializer();
1763 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1764 // Generate function that re-emits the declaration's initializer into the
1765 // threadprivate copy of the variable VD
1766 CodeGenFunction CtorCGF(CGM);
1767 FunctionArgList Args;
1768 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1769 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1770 ImplicitParamDecl::Other);
1771 Args.push_back(&Dst);
1772
1773 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1774 CGM.getContext().VoidPtrTy, Args);
1775 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1776 std::string Name = getName({"__kmpc_global_ctor_", ""});
1777 llvm::Function *Fn =
1778 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1779 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1780 Args, Loc, Loc);
1781 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1782 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1783 CGM.getContext().VoidPtrTy, Dst.getLocation());
1784 Address Arg = Address(ArgVal, VDAddr.getAlignment());
1785 Arg = CtorCGF.Builder.CreateElementBitCast(
1786 Arg, CtorCGF.ConvertTypeForMem(ASTTy));
1787 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1788 /*IsInitializer=*/true);
1789 ArgVal = CtorCGF.EmitLoadOfScalar(
1790 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
1791 CGM.getContext().VoidPtrTy, Dst.getLocation());
1792 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1793 CtorCGF.FinishFunction();
1794 Ctor = Fn;
1795 }
1796 if (VD->getType().isDestructedType() != QualType::DK_none) {
1797 // Generate function that emits destructor call for the threadprivate copy
1798 // of the variable VD
1799 CodeGenFunction DtorCGF(CGM);
1800 FunctionArgList Args;
1801 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
1802 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
1803 ImplicitParamDecl::Other);
1804 Args.push_back(&Dst);
1805
1806 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1807 CGM.getContext().VoidTy, Args);
1808 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1809 std::string Name = getName({"__kmpc_global_dtor_", ""});
1810 llvm::Function *Fn =
1811 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1812 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1813 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1814 Loc, Loc);
1815 // Create a scope with an artificial location for the body of this function.
1816 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1817 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1818 DtorCGF.GetAddrOfLocalVar(&Dst),
1819 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
1820 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
1821 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1822 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1823 DtorCGF.FinishFunction();
1824 Dtor = Fn;
1825 }
1826 // Do not emit init function if it is not required.
1827 if (!Ctor && !Dtor)
1828 return nullptr;
1829
1830 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1831 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1832 /*isVarArg=*/false)
1833 ->getPointerTo();
1834 // Copying constructor for the threadprivate variable.
1835 // Must be NULL - reserved by runtime, but currently it requires that this
1836 // parameter is always NULL. Otherwise it fires assertion.
1837 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1838 if (Ctor == nullptr) {
1839 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1840 /*isVarArg=*/false)
1841 ->getPointerTo();
1842 Ctor = llvm::Constant::getNullValue(CtorTy);
1843 }
1844 if (Dtor == nullptr) {
1845 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1846 /*isVarArg=*/false)
1847 ->getPointerTo();
1848 Dtor = llvm::Constant::getNullValue(DtorTy);
1849 }
1850 if (!CGF) {
1851 auto *InitFunctionTy =
1852 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1853 std::string Name = getName({"__omp_threadprivate_init_", ""});
1854 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1855 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
1856 CodeGenFunction InitCGF(CGM);
1857 FunctionArgList ArgList;
1858 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1859 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1860 Loc, Loc);
1861 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1862 InitCGF.FinishFunction();
1863 return InitFunction;
1864 }
1865 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1866 }
1867 return nullptr;
1868}
1869
1870bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
1871 llvm::GlobalVariable *Addr,
1872 bool PerformInit) {
1873 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
1874 !CGM.getLangOpts().OpenMPIsDevice)
1875 return false;
1876 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1877 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1878 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1879 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1880 HasRequiresUnifiedSharedMemory))
1881 return CGM.getLangOpts().OpenMPIsDevice;
1882 VD = VD->getDefinition(CGM.getContext());
1883 assert(VD && "Unknown VarDecl")((VD && "Unknown VarDecl") ? static_cast<void> (
0) : __assert_fail ("VD && \"Unknown VarDecl\"", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 1883, __PRETTY_FUNCTION__))
;
1884
1885 if (!DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
1886 return CGM.getLangOpts().OpenMPIsDevice;
1887
1888 QualType ASTTy = VD->getType();
1889 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
1890
1891 // Produce the unique prefix to identify the new target regions. We use
1892 // the source location of the variable declaration which we know to not
1893 // conflict with any target region.
1894 unsigned DeviceID;
1895 unsigned FileID;
1896 unsigned Line;
1897 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
1898 SmallString<128> Buffer, Out;
1899 {
1900 llvm::raw_svector_ostream OS(Buffer);
1901 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
1902 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
1903 }
1904
1905 const Expr *Init = VD->getAnyInitializer();
1906 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1907 llvm::Constant *Ctor;
1908 llvm::Constant *ID;
1909 if (CGM.getLangOpts().OpenMPIsDevice) {
1910 // Generate function that re-emits the declaration's initializer into
1911 // the threadprivate copy of the variable VD
1912 CodeGenFunction CtorCGF(CGM);
1913
1914 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1915 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1916 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1917 FTy, Twine(Buffer, "_ctor"), FI, Loc);
1918 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
1919 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1920 FunctionArgList(), Loc, Loc);
1921 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
1922 CtorCGF.EmitAnyExprToMem(Init,
1923 Address(Addr, CGM.getContext().getDeclAlign(VD)),
1924 Init->getType().getQualifiers(),
1925 /*IsInitializer=*/true);
1926 CtorCGF.FinishFunction();
1927 Ctor = Fn;
1928 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1929 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
1930 } else {
1931 Ctor = new llvm::GlobalVariable(
1932 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1933 llvm::GlobalValue::PrivateLinkage,
1934 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
1935 ID = Ctor;
1936 }
1937
1938 // Register the information for the entry associated with the constructor.
1939 Out.clear();
1940 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1941 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
1942 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
1943 }
1944 if (VD->getType().isDestructedType() != QualType::DK_none) {
1945 llvm::Constant *Dtor;
1946 llvm::Constant *ID;
1947 if (CGM.getLangOpts().OpenMPIsDevice) {
1948 // Generate function that emits destructor call for the threadprivate
1949 // copy of the variable VD
1950 CodeGenFunction DtorCGF(CGM);
1951
1952 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
1953 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1954 llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
1955 FTy, Twine(Buffer, "_dtor"), FI, Loc);
1956 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1957 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
1958 FunctionArgList(), Loc, Loc);
1959 // Create a scope with an artificial location for the body of this
1960 // function.
1961 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1962 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
1963 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1964 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1965 DtorCGF.FinishFunction();
1966 Dtor = Fn;
1967 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
1968 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
1969 } else {
1970 Dtor = new llvm::GlobalVariable(
1971 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1972 llvm::GlobalValue::PrivateLinkage,
1973 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
1974 ID = Dtor;
1975 }
1976 // Register the information for the entry associated with the destructor.
1977 Out.clear();
1978 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1979 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
1980 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
1981 }
1982 return CGM.getLangOpts().OpenMPIsDevice;
1983}
1984
1985Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1986 QualType VarType,
1987 StringRef Name) {
1988 std::string Suffix = getName({"artificial", ""});
1989 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1990 llvm::Value *GAddr =
1991 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
1992 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1993 CGM.getTarget().isTLSSupported()) {
1994 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
1995 return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
1996 }
1997 std::string CacheSuffix = getName({"cache", ""});
1998 llvm::Value *Args[] = {
1999 emitUpdateLocation(CGF, SourceLocation()),
2000 getThreadID(CGF, SourceLocation()),
2001 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2002 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2003 /*isSigned=*/false),
2004 getOrCreateInternalVariable(
2005 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2006 return Address(
2007 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2008 CGF.EmitRuntimeCall(
2009 OMPBuilder.getOrCreateRuntimeFunction(
2010 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
2011 Args),
2012 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2013 CGM.getContext().getTypeAlignInChars(VarType));
2014}
2015
2016void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
2017 const RegionCodeGenTy &ThenGen,
2018 const RegionCodeGenTy &ElseGen) {
2019 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2020
2021 // If the condition constant folds and can be elided, try to avoid emitting
2022 // the condition and the dead arm of the if/else.
2023 bool CondConstant;
2024 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2025 if (CondConstant)
2026 ThenGen(CGF);
2027 else
2028 ElseGen(CGF);
2029 return;
2030 }
2031
2032 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2033 // emit the conditional branch.
2034 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2035 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2036 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2037 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2038
2039 // Emit the 'then' code.
2040 CGF.EmitBlock(ThenBlock);
2041 ThenGen(CGF);
2042 CGF.EmitBranch(ContBlock);
2043 // Emit the 'else' code if present.
2044 // There is no need to emit line number for unconditional branch.
2045 (void)ApplyDebugLocation::CreateEmpty(CGF);
2046 CGF.EmitBlock(ElseBlock);
2047 ElseGen(CGF);
2048 // There is no need to emit line number for unconditional branch.
2049 (void)ApplyDebugLocation::CreateEmpty(CGF);
2050 CGF.EmitBranch(ContBlock);
2051 // Emit the continuation block for code after the if.
2052 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2053}
2054
2055void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2056 llvm::Function *OutlinedFn,
2057 ArrayRef<llvm::Value *> CapturedVars,
2058 const Expr *IfCond) {
2059 if (!CGF.HaveInsertPoint())
2060 return;
2061 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2062 auto &M = CGM.getModule();
2063 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2064 this](CodeGenFunction &CGF, PrePostActionTy &) {
2065 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2066 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2067 llvm::Value *Args[] = {
2068 RTLoc,
2069 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2070 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2071 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2072 RealArgs.append(std::begin(Args), std::end(Args));
2073 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2074
2075 llvm::FunctionCallee RTLFn =
2076 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2077 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2078 };
2079 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2080 this](CodeGenFunction &CGF, PrePostActionTy &) {
2081 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2082 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2083 // Build calls:
2084 // __kmpc_serialized_parallel(&Loc, GTid);
2085 llvm::Value *Args[] = {RTLoc, ThreadID};
2086 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2087 M, OMPRTL___kmpc_serialized_parallel),
2088 Args);
2089
2090 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2091 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2092 Address ZeroAddrBound =
2093 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2094 /*Name=*/".bound.zero.addr");
2095 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
2096 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2097 // ThreadId for serialized parallels is 0.
2098 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2099 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
2100 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2101
2102 // Ensure we do not inline the function. This is trivially true for the ones
2103 // passed to __kmpc_fork_call but the ones calles in serialized regions
2104 // could be inlined. This is not a perfect but it is closer to the invariant
2105 // we want, namely, every data environment starts with a new function.
2106 // TODO: We should pass the if condition to the runtime function and do the
2107 // handling there. Much cleaner code.
2108 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2109 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2110
2111 // __kmpc_end_serialized_parallel(&Loc, GTid);
2112 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2113 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2114 M, OMPRTL___kmpc_end_serialized_parallel),
2115 EndArgs);
2116 };
2117 if (IfCond) {
2118 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2119 } else {
2120 RegionCodeGenTy ThenRCG(ThenGen);
2121 ThenRCG(CGF);
2122 }
2123}
2124
2125// If we're inside an (outlined) parallel region, use the region info's
2126// thread-ID variable (it is passed in a first argument of the outlined function
2127// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2128// regular serial code region, get thread ID by calling kmp_int32
2129// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2130// return the address of that temp.
2131Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2132 SourceLocation Loc) {
2133 if (auto *OMPRegionInfo =
2134 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2135 if (OMPRegionInfo->getThreadIDVariable())
2136 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2137
2138 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2139 QualType Int32Ty =
2140 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2141 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2142 CGF.EmitStoreOfScalar(ThreadID,
2143 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2144
2145 return ThreadIDTemp;
2146}
2147
2148llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
2149 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2150 SmallString<256> Buffer;
2151 llvm::raw_svector_ostream Out(Buffer);
2152 Out << Name;
2153 StringRef RuntimeName = Out.str();
2154 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2155 if (Elem.second) {
2156 assert(Elem.second->getType()->getPointerElementType() == Ty &&((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2157, __PRETTY_FUNCTION__))
2157 "OMP internal variable has different type than requested")((Elem.second->getType()->getPointerElementType() == Ty
&& "OMP internal variable has different type than requested"
) ? static_cast<void> (0) : __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2157, __PRETTY_FUNCTION__))
;
2158 return &*Elem.second;
2159 }
2160
2161 return Elem.second = new llvm::GlobalVariable(
2162 CGM.getModule(), Ty, /*IsConstant*/ false,
2163 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2164 Elem.first(), /*InsertBefore=*/nullptr,
2165 llvm::GlobalValue::NotThreadLocal, AddressSpace);
2166}
2167
2168llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2169 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2170 std::string Name = getName({Prefix, "var"});
2171 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2172}
2173
2174namespace {
2175/// Common pre(post)-action for different OpenMP constructs.
2176class CommonActionTy final : public PrePostActionTy {
2177 llvm::FunctionCallee EnterCallee;
2178 ArrayRef<llvm::Value *> EnterArgs;
2179 llvm::FunctionCallee ExitCallee;
2180 ArrayRef<llvm::Value *> ExitArgs;
2181 bool Conditional;
2182 llvm::BasicBlock *ContBlock = nullptr;
2183
2184public:
2185 CommonActionTy(llvm::FunctionCallee EnterCallee,
2186 ArrayRef<llvm::Value *> EnterArgs,
2187 llvm::FunctionCallee ExitCallee,
2188 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2189 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2190 ExitArgs(ExitArgs), Conditional(Conditional) {}
2191 void Enter(CodeGenFunction &CGF) override {
2192 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2193 if (Conditional) {
2194 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2195 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2196 ContBlock = CGF.createBasicBlock("omp_if.end");
2197 // Generate the branch (If-stmt)
2198 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2199 CGF.EmitBlock(ThenBlock);
2200 }
2201 }
2202 void Done(CodeGenFunction &CGF) {
2203 // Emit the rest of blocks/branches
2204 CGF.EmitBranch(ContBlock);
2205 CGF.EmitBlock(ContBlock, true);
2206 }
2207 void Exit(CodeGenFunction &CGF) override {
2208 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2209 }
2210};
2211} // anonymous namespace
2212
2213void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2214 StringRef CriticalName,
2215 const RegionCodeGenTy &CriticalOpGen,
2216 SourceLocation Loc, const Expr *Hint) {
2217 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2218 // CriticalOpGen();
2219 // __kmpc_end_critical(ident_t *, gtid, Lock);
2220 // Prepare arguments and build a call to __kmpc_critical
2221 if (!CGF.HaveInsertPoint())
2222 return;
2223 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2224 getCriticalRegionLock(CriticalName)};
2225 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2226 std::end(Args));
2227 if (Hint) {
2228 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2229 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2230 }
2231 CommonActionTy Action(
2232 OMPBuilder.getOrCreateRuntimeFunction(
2233 CGM.getModule(),
2234 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2235 EnterArgs,
2236 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2237 OMPRTL___kmpc_end_critical),
2238 Args);
2239 CriticalOpGen.setAction(Action);
2240 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2241}
2242
2243void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2244 const RegionCodeGenTy &MasterOpGen,
2245 SourceLocation Loc) {
2246 if (!CGF.HaveInsertPoint())
2247 return;
2248 // if(__kmpc_master(ident_t *, gtid)) {
2249 // MasterOpGen();
2250 // __kmpc_end_master(ident_t *, gtid);
2251 // }
2252 // Prepare arguments and build a call to __kmpc_master
2253 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2254 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2255 CGM.getModule(), OMPRTL___kmpc_master),
2256 Args,
2257 OMPBuilder.getOrCreateRuntimeFunction(
2258 CGM.getModule(), OMPRTL___kmpc_end_master),
2259 Args,
2260 /*Conditional=*/true);
2261 MasterOpGen.setAction(Action);
2262 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2263 Action.Done(CGF);
2264}
2265
2266void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2267 SourceLocation Loc) {
2268 if (!CGF.HaveInsertPoint())
2269 return;
2270 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2271 OMPBuilder.createTaskyield(CGF.Builder);
2272 } else {
2273 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2274 llvm::Value *Args[] = {
2275 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2276 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
2277 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2278 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2279 Args);
2280 }
2281
2282 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2283 Region->emitUntiedSwitch(CGF);
2284}
2285
2286void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2287 const RegionCodeGenTy &TaskgroupOpGen,
2288 SourceLocation Loc) {
2289 if (!CGF.HaveInsertPoint())
2290 return;
2291 // __kmpc_taskgroup(ident_t *, gtid);
2292 // TaskgroupOpGen();
2293 // __kmpc_end_taskgroup(ident_t *, gtid);
2294 // Prepare arguments and build a call to __kmpc_taskgroup
2295 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2296 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2297 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2298 Args,
2299 OMPBuilder.getOrCreateRuntimeFunction(
2300 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2301 Args);
2302 TaskgroupOpGen.setAction(Action);
2303 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2304}
2305
2306/// Given an array of pointers to variables, project the address of a
2307/// given variable.
2308static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2309 unsigned Index, const VarDecl *Var) {
2310 // Pull out the pointer to the variable.
2311 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2312 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2313
2314 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
2315 Addr = CGF.Builder.CreateElementBitCast(
2316 Addr, CGF.ConvertTypeForMem(Var->getType()));
2317 return Addr;
2318}
2319
2320static llvm::Value *emitCopyprivateCopyFunction(
2321 CodeGenModule &CGM, llvm::Type *ArgsType,
2322 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2323 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2324 SourceLocation Loc) {
2325 ASTContext &C = CGM.getContext();
2326 // void copy_func(void *LHSArg, void *RHSArg);
2327 FunctionArgList Args;
2328 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2329 ImplicitParamDecl::Other);
2330 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
2331 ImplicitParamDecl::Other);
2332 Args.push_back(&LHSArg);
2333 Args.push_back(&RHSArg);
2334 const auto &CGFI =
2335 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
2336 std::string Name =
2337 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
2338 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
2339 llvm::GlobalValue::InternalLinkage, Name,
2340 &CGM.getModule());
2341 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
2342 Fn->setDoesNotRecurse();
2343 CodeGenFunction CGF(CGM);
2344 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2345 // Dest = (void*[n])(LHSArg);
2346 // Src = (void*[n])(RHSArg);
2347 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2348 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2349 ArgsType), CGF.getPointerAlign());
2350 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2351 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2352 ArgsType), CGF.getPointerAlign());
2353 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2354 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2355 // ...
2356 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2357 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2358 const auto *DestVar =
2359 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2360 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2361
2362 const auto *SrcVar =
2363 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2364 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2365
2366 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2367 QualType Type = VD->getType();
2368 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2369 }
2370 CGF.FinishFunction();
2371 return Fn;
2372}
2373
2374void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2375 const RegionCodeGenTy &SingleOpGen,
2376 SourceLocation Loc,
2377 ArrayRef<const Expr *> CopyprivateVars,
2378 ArrayRef<const Expr *> SrcExprs,
2379 ArrayRef<const Expr *> DstExprs,
2380 ArrayRef<const Expr *> AssignmentOps) {
2381 if (!CGF.HaveInsertPoint())
2382 return;
2383 assert(CopyprivateVars.size() == SrcExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2385, __PRETTY_FUNCTION__))
2384 CopyprivateVars.size() == DstExprs.size() &&((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2385, __PRETTY_FUNCTION__))
2385 CopyprivateVars.size() == AssignmentOps.size())((CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars
.size() == DstExprs.size() && CopyprivateVars.size() ==
AssignmentOps.size()) ? static_cast<void> (0) : __assert_fail
("CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2385, __PRETTY_FUNCTION__))
;
2386 ASTContext &C = CGM.getContext();
2387 // int32 did_it = 0;
2388 // if(__kmpc_single(ident_t *, gtid)) {
2389 // SingleOpGen();
2390 // __kmpc_end_single(ident_t *, gtid);
2391 // did_it = 1;
2392 // }
2393 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2394 // <copy_func>, did_it);
2395
2396 Address DidIt = Address::invalid();
2397 if (!CopyprivateVars.empty()) {
2398 // int32 did_it = 0;
2399 QualType KmpInt32Ty =
2400 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2401 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2402 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2403 }
2404 // Prepare arguments and build a call to __kmpc_single
2405 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2406 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2407 CGM.getModule(), OMPRTL___kmpc_single),
2408 Args,
2409 OMPBuilder.getOrCreateRuntimeFunction(
2410 CGM.getModule(), OMPRTL___kmpc_end_single),
2411 Args,
2412 /*Conditional=*/true);
2413 SingleOpGen.setAction(Action);
2414 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2415 if (DidIt.isValid()) {
2416 // did_it = 1;
2417 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2418 }
2419 Action.Done(CGF);
2420 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2421 // <copy_func>, did_it);
2422 if (DidIt.isValid()) {
2423 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2424 QualType CopyprivateArrayTy = C.getConstantArrayType(
2425 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
2426 /*IndexTypeQuals=*/0);
2427 // Create a list of all private variables for copyprivate.
2428 Address CopyprivateList =
2429 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2430 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2431 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2432 CGF.Builder.CreateStore(
2433 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2434 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2435 CGF.VoidPtrTy),
2436 Elem);
2437 }
2438 // Build function that copies private values from single region to all other
2439 // threads in the corresponding parallel region.
2440 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2441 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
2442 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
2443 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2444 Address CL =
2445 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2446 CGF.VoidPtrTy);
2447 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2448 llvm::Value *Args[] = {
2449 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2450 getThreadID(CGF, Loc), // i32 <gtid>
2451 BufSize, // size_t <buf_size>
2452 CL.getPointer(), // void *<copyprivate list>
2453 CpyFn, // void (*) (void *, void *) <copy_func>
2454 DidItVal // i32 did_it
2455 };
2456 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2457 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2458 Args);
2459 }
2460}
2461
2462void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2463 const RegionCodeGenTy &OrderedOpGen,
2464 SourceLocation Loc, bool IsThreads) {
2465 if (!CGF.HaveInsertPoint())
2466 return;
2467 // __kmpc_ordered(ident_t *, gtid);
2468 // OrderedOpGen();
2469 // __kmpc_end_ordered(ident_t *, gtid);
2470 // Prepare arguments and build a call to __kmpc_ordered
2471 if (IsThreads) {
2472 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2473 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2474 CGM.getModule(), OMPRTL___kmpc_ordered),
2475 Args,
2476 OMPBuilder.getOrCreateRuntimeFunction(
2477 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2478 Args);
2479 OrderedOpGen.setAction(Action);
2480 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2481 return;
2482 }
2483 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2484}
2485
2486unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2487 unsigned Flags;
2488 if (Kind == OMPD_for)
2489 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2490 else if (Kind == OMPD_sections)
2491 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2492 else if (Kind == OMPD_single)
2493 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2494 else if (Kind == OMPD_barrier)
2495 Flags = OMP_IDENT_BARRIER_EXPL;
2496 else
2497 Flags = OMP_IDENT_BARRIER_IMPL;
2498 return Flags;
2499}
2500
2501void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2502 CodeGenFunction &CGF, const OMPLoopDirective &S,
2503 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2504 // Check if the loop directive is actually a doacross loop directive. In this
2505 // case choose static, 1 schedule.
2506 if (llvm::any_of(
2507 S.getClausesOfKind<OMPOrderedClause>(),
2508 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2509 ScheduleKind = OMPC_SCHEDULE_static;
2510 // Chunk size is 1 in this case.
2511 llvm::APInt ChunkSize(32, 1);
2512 ChunkExpr = IntegerLiteral::Create(
2513 CGF.getContext(), ChunkSize,
2514 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2515 SourceLocation());
2516 }
2517}
2518
2519void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2520 OpenMPDirectiveKind Kind, bool EmitChecks,
2521 bool ForceSimpleCall) {
2522 // Check if we should use the OMPBuilder
2523 auto *OMPRegionInfo =
2524 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2525 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2526 CGF.Builder.restoreIP(OMPBuilder.createBarrier(
2527 CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
2528 return;
2529 }
2530
2531 if (!CGF.HaveInsertPoint())
2532 return;
2533 // Build call __kmpc_cancel_barrier(loc, thread_id);
2534 // Build call __kmpc_barrier(loc, thread_id);
2535 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2536 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2537 // thread_id);
2538 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2539 getThreadID(CGF, Loc)};
2540 if (OMPRegionInfo) {
2541 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2542 llvm::Value *Result = CGF.EmitRuntimeCall(
2543 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2544 OMPRTL___kmpc_cancel_barrier),
2545 Args);
2546 if (EmitChecks) {
2547 // if (__kmpc_cancel_barrier()) {
2548 // exit from construct;
2549 // }
2550 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2551 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2552 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2553 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2554 CGF.EmitBlock(ExitBB);
2555 // exit from construct;
2556 CodeGenFunction::JumpDest CancelDestination =
2557 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2558 CGF.EmitBranchThroughCleanup(CancelDestination);
2559 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2560 }
2561 return;
2562 }
2563 }
2564 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2565 CGM.getModule(), OMPRTL___kmpc_barrier),
2566 Args);
2567}
2568
2569/// Map the OpenMP loop schedule to the runtime enumeration.
2570static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2571 bool Chunked, bool Ordered) {
2572 switch (ScheduleKind) {
2573 case OMPC_SCHEDULE_static:
2574 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2575 : (Ordered ? OMP_ord_static : OMP_sch_static);
2576 case OMPC_SCHEDULE_dynamic:
2577 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2578 case OMPC_SCHEDULE_guided:
2579 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2580 case OMPC_SCHEDULE_runtime:
2581 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2582 case OMPC_SCHEDULE_auto:
2583 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2584 case OMPC_SCHEDULE_unknown:
2585 assert(!Chunked && "chunk was specified but schedule kind not known")((!Chunked && "chunk was specified but schedule kind not known"
) ? static_cast<void> (0) : __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2585, __PRETTY_FUNCTION__))
;
2586 return Ordered ? OMP_ord_static : OMP_sch_static;
2587 }
2588 llvm_unreachable("Unexpected runtime schedule")::llvm::llvm_unreachable_internal("Unexpected runtime schedule"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2588)
;
2589}
2590
2591/// Map the OpenMP distribute schedule to the runtime enumeration.
2592static OpenMPSchedType
2593getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2594 // only static is allowed for dist_schedule
2595 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2596}
2597
2598bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2599 bool Chunked) const {
2600 OpenMPSchedType Schedule =
2601 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2602 return Schedule == OMP_sch_static;
2603}
2604
2605bool CGOpenMPRuntime::isStaticNonchunked(
2606 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2607 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2608 return Schedule == OMP_dist_sch_static;
2609}
2610
2611bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2612 bool Chunked) const {
2613 OpenMPSchedType Schedule =
2614 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2615 return Schedule == OMP_sch_static_chunked;
2616}
2617
2618bool CGOpenMPRuntime::isStaticChunked(
2619 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2620 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2621 return Schedule == OMP_dist_sch_static_chunked;
2622}
2623
2624bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2625 OpenMPSchedType Schedule =
2626 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2627 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here")((Schedule != OMP_sch_static_chunked && "cannot be chunked here"
) ? static_cast<void> (0) : __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2627, __PRETTY_FUNCTION__))
;
2628 return Schedule != OMP_sch_static;
2629}
2630
2631static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2632 OpenMPScheduleClauseModifier M1,
2633 OpenMPScheduleClauseModifier M2) {
2634 int Modifier = 0;
2635 switch (M1) {
2636 case OMPC_SCHEDULE_MODIFIER_monotonic:
2637 Modifier = OMP_sch_modifier_monotonic;
2638 break;
2639 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2640 Modifier = OMP_sch_modifier_nonmonotonic;
2641 break;
2642 case OMPC_SCHEDULE_MODIFIER_simd:
2643 if (Schedule == OMP_sch_static_chunked)
2644 Schedule = OMP_sch_static_balanced_chunked;
2645 break;
2646 case OMPC_SCHEDULE_MODIFIER_last:
2647 case OMPC_SCHEDULE_MODIFIER_unknown:
2648 break;
2649 }
2650 switch (M2) {
2651 case OMPC_SCHEDULE_MODIFIER_monotonic:
2652 Modifier = OMP_sch_modifier_monotonic;
2653 break;
2654 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2655 Modifier = OMP_sch_modifier_nonmonotonic;
2656 break;
2657 case OMPC_SCHEDULE_MODIFIER_simd:
2658 if (Schedule == OMP_sch_static_chunked)
2659 Schedule = OMP_sch_static_balanced_chunked;
2660 break;
2661 case OMPC_SCHEDULE_MODIFIER_last:
2662 case OMPC_SCHEDULE_MODIFIER_unknown:
2663 break;
2664 }
2665 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2666 // If the static schedule kind is specified or if the ordered clause is
2667 // specified, and if the nonmonotonic modifier is not specified, the effect is
2668 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2669 // modifier is specified, the effect is as if the nonmonotonic modifier is
2670 // specified.
2671 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2672 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2673 Schedule == OMP_sch_static_balanced_chunked ||
2674 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2675 Schedule == OMP_dist_sch_static_chunked ||
2676 Schedule == OMP_dist_sch_static))
2677 Modifier = OMP_sch_modifier_nonmonotonic;
2678 }
2679 return Schedule | Modifier;
2680}
2681
2682void CGOpenMPRuntime::emitForDispatchInit(
2683 CodeGenFunction &CGF, SourceLocation Loc,
2684 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2685 bool Ordered, const DispatchRTInput &DispatchValues) {
2686 if (!CGF.HaveInsertPoint())
2687 return;
2688 OpenMPSchedType Schedule = getRuntimeSchedule(
2689 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
2690 assert(Ordered ||((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2693, __PRETTY_FUNCTION__))
2691 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2693, __PRETTY_FUNCTION__))
2692 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2693, __PRETTY_FUNCTION__))
2693 Schedule != OMP_sch_static_balanced_chunked))((Ordered || (Schedule != OMP_sch_static && Schedule !=
OMP_sch_static_chunked && Schedule != OMP_ord_static
&& Schedule != OMP_ord_static_chunked && Schedule
!= OMP_sch_static_balanced_chunked)) ? static_cast<void>
(0) : __assert_fail ("Ordered || (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && Schedule != OMP_sch_static_balanced_chunked)"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2693, __PRETTY_FUNCTION__))
;
2694 // Call __kmpc_dispatch_init(
2695 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2696 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2697 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2698
2699 // If the Chunk was not specified in the clause - use default value 1.
2700 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2701 : CGF.Builder.getIntN(IVSize, 1);
2702 llvm::Value *Args[] = {
2703 emitUpdateLocation(CGF, Loc),
2704 getThreadID(CGF, Loc),
2705 CGF.Builder.getInt32(addMonoNonMonoModifier(
2706 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
2707 DispatchValues.LB, // Lower
2708 DispatchValues.UB, // Upper
2709 CGF.Builder.getIntN(IVSize, 1), // Stride
2710 Chunk // Chunk
2711 };
2712 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2713}
2714
2715static void emitForStaticInitCall(
2716 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2717 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2718 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2719 const CGOpenMPRuntime::StaticRTInput &Values) {
2720 if (!CGF.HaveInsertPoint())
2721 return;
2722
2723 assert(!Values.Ordered)((!Values.Ordered) ? static_cast<void> (0) : __assert_fail
("!Values.Ordered", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2723, __PRETTY_FUNCTION__))
;
2724 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2728, __PRETTY_FUNCTION__))
2725 Schedule == OMP_sch_static_balanced_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2728, __PRETTY_FUNCTION__))
2726 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2728, __PRETTY_FUNCTION__))
2727 Schedule == OMP_dist_sch_static ||((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2728, __PRETTY_FUNCTION__))
2728 Schedule == OMP_dist_sch_static_chunked)((Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked
|| Schedule == OMP_sch_static_balanced_chunked || Schedule ==
OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule
== OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked
) ? static_cast<void> (0) : __assert_fail ("Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static || Schedule == OMP_dist_sch_static_chunked"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2728, __PRETTY_FUNCTION__))
;
2729
2730 // Call __kmpc_for_static_init(
2731 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2732 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2733 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2734 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2735 llvm::Value *Chunk = Values.Chunk;
2736 if (Chunk == nullptr) {
2737 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2738 Schedule == OMP_dist_sch_static) &&(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
2739 "expected static non-chunked schedule")(((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
Schedule == OMP_dist_sch_static) && "expected static non-chunked schedule"
) ? static_cast<void> (0) : __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2739, __PRETTY_FUNCTION__))
;
2740 // If the Chunk was not specified in the clause - use default value 1.
2741 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2742 } else {
2743 assert((Schedule == OMP_sch_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2747, __PRETTY_FUNCTION__))
2744 Schedule == OMP_sch_static_balanced_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2747, __PRETTY_FUNCTION__))
2745 Schedule == OMP_ord_static_chunked ||(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2747, __PRETTY_FUNCTION__))
2746 Schedule == OMP_dist_sch_static_chunked) &&(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2747, __PRETTY_FUNCTION__))
2747 "expected static chunked schedule")(((Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked
|| Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked
) && "expected static chunked schedule") ? static_cast
<void> (0) : __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2747, __PRETTY_FUNCTION__))
;
2748 }
2749 llvm::Value *Args[] = {
2750 UpdateLocation,
2751 ThreadId,
2752 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2753 M2)), // Schedule type
2754 Values.IL.getPointer(), // &isLastIter
2755 Values.LB.getPointer(), // &LB
2756 Values.UB.getPointer(), // &UB
2757 Values.ST.getPointer(), // &Stride
2758 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2759 Chunk // Chunk
2760 };
2761 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2762}
2763
2764void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2765 SourceLocation Loc,
2766 OpenMPDirectiveKind DKind,
2767 const OpenMPScheduleTy &ScheduleKind,
2768 const StaticRTInput &Values) {
2769 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
2770 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
2771 assert(isOpenMPWorksharingDirective(DKind) &&((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2772, __PRETTY_FUNCTION__))
2772 "Expected loop-based or sections-based directive.")((isOpenMPWorksharingDirective(DKind) && "Expected loop-based or sections-based directive."
) ? static_cast<void> (0) : __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2772, __PRETTY_FUNCTION__))
;
2773 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2774 isOpenMPLoopDirective(DKind)
2775 ? OMP_IDENT_WORK_LOOP
2776 : OMP_IDENT_WORK_SECTIONS);
2777 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2778 llvm::FunctionCallee StaticInitFunction =
2779 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2780 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2781 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2782 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
2783}
2784
2785void CGOpenMPRuntime::emitDistributeStaticInit(
2786 CodeGenFunction &CGF, SourceLocation Loc,
2787 OpenMPDistScheduleClauseKind SchedKind,
2788 const CGOpenMPRuntime::StaticRTInput &Values) {
2789 OpenMPSchedType ScheduleNum =
2790 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
2791 llvm::Value *UpdatedLocation =
2792 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2793 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2794 llvm::FunctionCallee StaticInitFunction =
2795 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
2796 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2797 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
2798 OMPC_SCHEDULE_MODIFIER_unknown, Values);
2799}
2800
2801void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2802 SourceLocation Loc,
2803 OpenMPDirectiveKind DKind) {
2804 if (!CGF.HaveInsertPoint())
2805 return;
2806 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2807 llvm::Value *Args[] = {
2808 emitUpdateLocation(CGF, Loc,
2809 isOpenMPDistributeDirective(DKind)
2810 ? OMP_IDENT_WORK_DISTRIBUTE
2811 : isOpenMPLoopDirective(DKind)
2812 ? OMP_IDENT_WORK_LOOP
2813 : OMP_IDENT_WORK_SECTIONS),
2814 getThreadID(CGF, Loc)};
2815 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
2816 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2817 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2818 Args);
2819}
2820
2821void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2822 SourceLocation Loc,
2823 unsigned IVSize,
2824 bool IVSigned) {
2825 if (!CGF.HaveInsertPoint())
2826 return;
2827 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2828 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2829 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2830}
2831
2832llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2833 SourceLocation Loc, unsigned IVSize,
2834 bool IVSigned, Address IL,
2835 Address LB, Address UB,
2836 Address ST) {
2837 // Call __kmpc_dispatch_next(
2838 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2839 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2840 // kmp_int[32|64] *p_stride);
2841 llvm::Value *Args[] = {
2842 emitUpdateLocation(CGF, Loc),
2843 getThreadID(CGF, Loc),
2844 IL.getPointer(), // &isLastIter
2845 LB.getPointer(), // &Lower
2846 UB.getPointer(), // &Upper
2847 ST.getPointer() // &Stride
2848 };
2849 llvm::Value *Call =
2850 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2851 return CGF.EmitScalarConversion(
2852 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2853 CGF.getContext().BoolTy, Loc);
2854}
2855
2856void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2857 llvm::Value *NumThreads,
2858 SourceLocation Loc) {
2859 if (!CGF.HaveInsertPoint())
2860 return;
2861 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2862 llvm::Value *Args[] = {
2863 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2864 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2865 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2866 CGM.getModule(), OMPRTL___kmpc_push_num_threads),
2867 Args);
2868}
2869
2870void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2871 ProcBindKind ProcBind,
2872 SourceLocation Loc) {
2873 if (!CGF.HaveInsertPoint())
2874 return;
2875 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.")((ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."
) ? static_cast<void> (0) : __assert_fail ("ProcBind != OMP_PROC_BIND_unknown && \"Unsupported proc_bind value.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2875, __PRETTY_FUNCTION__))
;
2876 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2877 llvm::Value *Args[] = {
2878 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2879 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
2880 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2881 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2882 Args);
2883}
2884
2885void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2886 SourceLocation Loc, llvm::AtomicOrdering AO) {
2887 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2888 OMPBuilder.createFlush(CGF.Builder);
2889 } else {
2890 if (!CGF.HaveInsertPoint())
2891 return;
2892 // Build call void __kmpc_flush(ident_t *loc)
2893 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2894 CGM.getModule(), OMPRTL___kmpc_flush),
2895 emitUpdateLocation(CGF, Loc));
2896 }
2897}
2898
2899namespace {
2900/// Indexes of fields for type kmp_task_t.
2901enum KmpTaskTFields {
2902 /// List of shared variables.
2903 KmpTaskTShareds,
2904 /// Task routine.
2905 KmpTaskTRoutine,
2906 /// Partition id for the untied tasks.
2907 KmpTaskTPartId,
2908 /// Function with call of destructors for private variables.
2909 Data1,
2910 /// Task priority.
2911 Data2,
2912 /// (Taskloops only) Lower bound.
2913 KmpTaskTLowerBound,
2914 /// (Taskloops only) Upper bound.
2915 KmpTaskTUpperBound,
2916 /// (Taskloops only) Stride.
2917 KmpTaskTStride,
2918 /// (Taskloops only) Is last iteration flag.
2919 KmpTaskTLastIter,
2920 /// (Taskloops only) Reduction data.
2921 KmpTaskTReductions,
2922};
2923} // anonymous namespace
2924
2925bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
2926 return OffloadEntriesTargetRegion.empty() &&
2927 OffloadEntriesDeviceGlobalVar.empty();
2928}
2929
2930/// Initialize target region entry.
2931void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2932 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2933 StringRef ParentName, unsigned LineNum,
2934 unsigned Order) {
2935 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2937, __PRETTY_FUNCTION__))
2936 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2937, __PRETTY_FUNCTION__))
2937 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2937, __PRETTY_FUNCTION__))
;
2938 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
2939 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
2940 OMPTargetRegionEntryTargetRegion);
2941 ++OffloadingEntriesNum;
2942}
2943
2944void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2945 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
2946 StringRef ParentName, unsigned LineNum,
2947 llvm::Constant *Addr, llvm::Constant *ID,
2948 OMPTargetRegionEntryKind Flags) {
2949 // If we are emitting code for a target, the entry is already initialized,
2950 // only has to be registered.
2951 if (CGM.getLangOpts().OpenMPIsDevice) {
2952 // This could happen if the device compilation is invoked standalone.
2953 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum))
2954 initializeTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2955 OffloadingEntriesNum);
2956 auto &Entry =
2957 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
2958 Entry.setAddress(Addr);
2959 Entry.setID(ID);
2960 Entry.setFlags(Flags);
2961 } else {
2962 if (Flags ==
2963 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion &&
2964 hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2965 /*IgnoreAddressId*/ true))
2966 return;
2967 assert(!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2968, __PRETTY_FUNCTION__))
2968 "Target region entry already registered!")((!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum
) && "Target region entry already registered!") ? static_cast
<void> (0) : __assert_fail ("!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && \"Target region entry already registered!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 2968, __PRETTY_FUNCTION__))
;
2969 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
2970 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
2971 ++OffloadingEntriesNum;
2972 }
2973}
2974
2975bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2976 unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum,
2977 bool IgnoreAddressId) const {
2978 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2979 if (PerDevice == OffloadEntriesTargetRegion.end())
2980 return false;
2981 auto PerFile = PerDevice->second.find(FileID);
2982 if (PerFile == PerDevice->second.end())
2983 return false;
2984 auto PerParentName = PerFile->second.find(ParentName);
2985 if (PerParentName == PerFile->second.end())
2986 return false;
2987 auto PerLine = PerParentName->second.find(LineNum);
2988 if (PerLine == PerParentName->second.end())
2989 return false;
2990 // Fail if this entry is already registered.
2991 if (!IgnoreAddressId &&
2992 (PerLine->second.getAddress() || PerLine->second.getID()))
2993 return false;
2994 return true;
2995}
2996
2997void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
2998 const OffloadTargetRegionEntryInfoActTy &Action) {
2999 // Scan all target region entries and perform the provided action.
3000 for (const auto &D : OffloadEntriesTargetRegion)
3001 for (const auto &F : D.second)
3002 for (const auto &P : F.second)
3003 for (const auto &L : P.second)
3004 Action(D.first, F.first, P.first(), L.first, L.second);
3005}
3006
3007void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3008 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3009 OMPTargetGlobalVarEntryKind Flags,
3010 unsigned Order) {
3011 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3013, __PRETTY_FUNCTION__))
3012 "only required for the device "((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3013, __PRETTY_FUNCTION__))
3013 "code generation.")((CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
"only required for the device " "code generation.") ? static_cast
<void> (0) : __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3013, __PRETTY_FUNCTION__))
;
3014 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3015 ++OffloadingEntriesNum;
3016}
3017
3018void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3019 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3020 CharUnits VarSize,
3021 OMPTargetGlobalVarEntryKind Flags,
3022 llvm::GlobalValue::LinkageTypes Linkage) {
3023 if (CGM.getLangOpts().OpenMPIsDevice) {
3024 // This could happen if the device compilation is invoked standalone.
3025 if (!hasDeviceGlobalVarEntryInfo(VarName))
3026 initializeDeviceGlobalVarEntryInfo(VarName, Flags, OffloadingEntriesNum);
3027 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3028 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3029, __PRETTY_FUNCTION__))
3029 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3029, __PRETTY_FUNCTION__))
;
3030 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3031 if (Entry.getVarSize().isZero()) {
3032 Entry.setVarSize(VarSize);
3033 Entry.setLinkage(Linkage);
3034 }
3035 return;
3036 }
3037 Entry.setVarSize(VarSize);
3038 Entry.setLinkage(Linkage);
3039 Entry.setAddress(Addr);
3040 } else {
3041 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3042 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3043 assert(Entry.isValid() && Entry.getFlags() == Flags &&((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3044, __PRETTY_FUNCTION__))
3044 "Entry not initialized!")((Entry.isValid() && Entry.getFlags() == Flags &&
"Entry not initialized!") ? static_cast<void> (0) : __assert_fail
("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3044, __PRETTY_FUNCTION__))
;
3045 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3046, __PRETTY_FUNCTION__))
3046 "Resetting with the new address.")(((!Entry.getAddress() || Entry.getAddress() == Addr) &&
"Resetting with the new address.") ? static_cast<void>
(0) : __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3046, __PRETTY_FUNCTION__))
;
3047 if (Entry.getVarSize().isZero()) {
3048 Entry.setVarSize(VarSize);
3049 Entry.setLinkage(Linkage);
3050 }
3051 return;
3052 }
3053 OffloadEntriesDeviceGlobalVar.try_emplace(
3054 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3055 ++OffloadingEntriesNum;
3056 }
3057}
3058
3059void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3060 actOnDeviceGlobalVarEntriesInfo(
3061 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3062 // Scan all target region entries and perform the provided action.
3063 for (const auto &E : OffloadEntriesDeviceGlobalVar)
3064 Action(E.getKey(), E.getValue());
3065}
3066
3067void CGOpenMPRuntime::createOffloadEntry(
3068 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3069 llvm::GlobalValue::LinkageTypes Linkage) {
3070 StringRef Name = Addr->getName();
3071 llvm::Module &M = CGM.getModule();
3072 llvm::LLVMContext &C = M.getContext();
3073
3074 // Create constant string with the name.
3075 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3076
3077 std::string StringName = getName({"omp_offloading", "entry_name"});
3078 auto *Str = new llvm::GlobalVariable(
3079 M, StrPtrInit->getType(), /*isConstant=*/true,
3080 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3081 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3082
3083 llvm::Constant *Data[] = {
3084 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(ID, CGM.VoidPtrTy),
3085 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, CGM.Int8PtrTy),
3086 llvm::ConstantInt::get(CGM.SizeTy, Size),
3087 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3088 llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3089 std::string EntryName = getName({"omp_offloading", "entry", ""});
3090 llvm::GlobalVariable *Entry = createGlobalStruct(
3091 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
3092 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
3093
3094 // The entry has to be created in the section the linker expects it to be.
3095 Entry->setSection("omp_offloading_entries");
3096}
3097
3098void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3099 // Emit the offloading entries and metadata so that the device codegen side
3100 // can easily figure out what to emit. The produced metadata looks like
3101 // this:
3102 //
3103 // !omp_offload.info = !{!1, ...}
3104 //
3105 // Right now we only generate metadata for function that contain target
3106 // regions.
3107
3108 // If we are in simd mode or there are no entries, we don't need to do
3109 // anything.
3110 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
3111 return;
3112
3113 llvm::Module &M = CGM.getModule();
3114 llvm::LLVMContext &C = M.getContext();
3115 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
3116 SourceLocation, StringRef>,
3117 16>
3118 OrderedEntries(OffloadEntriesInfoManager.size());
3119 llvm::SmallVector<StringRef, 16> ParentFunctions(
3120 OffloadEntriesInfoManager.size());
3121
3122 // Auxiliary methods to create metadata values and strings.
3123 auto &&GetMDInt = [this](unsigned V) {
3124 return llvm::ConstantAsMetadata::get(
3125 llvm::ConstantInt::get(CGM.Int32Ty, V));
3126 };
3127
3128 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3129
3130 // Create the offloading info metadata node.
3131 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3132
3133 // Create function that emits metadata for each target region entry;
3134 auto &&TargetRegionMetadataEmitter =
3135 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
3136 &GetMDString](
3137 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3138 unsigned Line,
3139 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3140 // Generate metadata for target regions. Each entry of this metadata
3141 // contains:
3142 // - Entry 0 -> Kind of this type of metadata (0).
3143 // - Entry 1 -> Device ID of the file where the entry was identified.
3144 // - Entry 2 -> File ID of the file where the entry was identified.
3145 // - Entry 3 -> Mangled name of the function where the entry was
3146 // identified.
3147 // - Entry 4 -> Line in the file where the entry was identified.
3148 // - Entry 5 -> Order the entry was created.
3149 // The first element of the metadata node is the kind.
3150 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3151 GetMDInt(FileID), GetMDString(ParentName),
3152 GetMDInt(Line), GetMDInt(E.getOrder())};
3153
3154 SourceLocation Loc;
3155 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3156 E = CGM.getContext().getSourceManager().fileinfo_end();
3157 I != E; ++I) {
3158 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
3159 I->getFirst()->getUniqueID().getFile() == FileID) {
3160 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3161 I->getFirst(), Line, 1);
3162 break;
3163 }
3164 }
3165 // Save this entry in the right position of the ordered entries array.
3166 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
3167 ParentFunctions[E.getOrder()] = ParentName;
3168
3169 // Add metadata to the named metadata node.
3170 MD->addOperand(llvm::MDNode::get(C, Ops));
3171 };
3172
3173 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3174 TargetRegionMetadataEmitter);
3175
3176 // Create function that emits metadata for each device global variable entry;
3177 auto &&DeviceGlobalVarMetadataEmitter =
3178 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3179 MD](StringRef MangledName,
3180 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3181 &E) {
3182 // Generate metadata for global variables. Each entry of this metadata
3183 // contains:
3184 // - Entry 0 -> Kind of this type of metadata (1).
3185 // - Entry 1 -> Mangled name of the variable.
3186 // - Entry 2 -> Declare target kind.
3187 // - Entry 3 -> Order the entry was created.
3188 // The first element of the metadata node is the kind.
3189 llvm::Metadata *Ops[] = {
3190 GetMDInt(E.getKind()), GetMDString(MangledName),
3191 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3192
3193 // Save this entry in the right position of the ordered entries array.
3194 OrderedEntries[E.getOrder()] =
3195 std::make_tuple(&E, SourceLocation(), MangledName);
3196
3197 // Add metadata to the named metadata node.
3198 MD->addOperand(llvm::MDNode::get(C, Ops));
3199 };
3200
3201 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3202 DeviceGlobalVarMetadataEmitter);
3203
3204 for (const auto &E : OrderedEntries) {
3205 assert(std::get<0>(E) && "All ordered entries must exist!")((std::get<0>(E) && "All ordered entries must exist!"
) ? static_cast<void> (0) : __assert_fail ("std::get<0>(E) && \"All ordered entries must exist!\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3205, __PRETTY_FUNCTION__))
;
3206 if (const auto *CE =
3207 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3208 std::get<0>(E))) {
3209 if (!CE->getID() || !CE->getAddress()) {
3210 // Do not blame the entry if the parent funtion is not emitted.
3211 StringRef FnName = ParentFunctions[CE->getOrder()];
3212 if (!CGM.GetGlobalValue(FnName))
3213 continue;
3214 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3215 DiagnosticsEngine::Error,
3216 "Offloading entry for target region in %0 is incorrect: either the "
3217 "address or the ID is invalid.");
3218 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
3219 continue;
3220 }
3221 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
3222 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3223 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
3224 OffloadEntryInfoDeviceGlobalVar>(
3225 std::get<0>(E))) {
3226 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3227 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3228 CE->getFlags());
3229 switch (Flags) {
3230 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3231 if (CGM.getLangOpts().OpenMPIsDevice &&
3232 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
3233 continue;
3234 if (!CE->getAddress()) {
3235 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3236 DiagnosticsEngine::Error, "Offloading entry for declare target "
3237 "variable %0 is incorrect: the "
3238 "address is invalid.");
3239 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
3240 continue;
3241 }
3242 // The vaiable has no definition - no need to add the entry.
3243 if (CE->getVarSize().isZero())
3244 continue;
3245 break;
3246 }
3247 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3248 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3250, __PRETTY_FUNCTION__))
3249 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3250, __PRETTY_FUNCTION__))
3250 "Declaret target link address is set.")((((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress
()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress
())) && "Declaret target link address is set.") ? static_cast
<void> (0) : __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3250, __PRETTY_FUNCTION__))
;
3251 if (CGM.getLangOpts().OpenMPIsDevice)
3252 continue;
3253 if (!CE->getAddress()) {
3254 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3255 DiagnosticsEngine::Error,
3256 "Offloading entry for declare target variable is incorrect: the "
3257 "address is invalid.");
3258 CGM.getDiags().Report(DiagID);
3259 continue;
3260 }
3261 break;
3262 }
3263 createOffloadEntry(CE->getAddress(), CE->getAddress(),
3264 CE->getVarSize().getQuantity(), Flags,
3265 CE->getLinkage());
3266 } else {
3267 llvm_unreachable("Unsupported entry kind.")::llvm::llvm_unreachable_internal("Unsupported entry kind.", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3267)
;
3268 }
3269 }
3270}
3271
3272/// Loads all the offload entries information from the host IR
3273/// metadata.
3274void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3275 // If we are in target mode, load the metadata from the host IR. This code has
3276 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
3277
3278 if (!CGM.getLangOpts().OpenMPIsDevice)
3279 return;
3280
3281 if (CGM.getLangOpts().OMPHostIRFile.empty())
3282 return;
3283
3284 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3285 if (auto EC = Buf.getError()) {
3286 CGM.getDiags().Report(diag::err_cannot_open_file)
3287 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3288 return;
3289 }
3290
3291 llvm::LLVMContext C;
3292 auto ME = expectedToErrorOrAndEmitErrors(
3293 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3294
3295 if (auto EC = ME.getError()) {
3296 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3297 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
3298 CGM.getDiags().Report(DiagID)
3299 << CGM.getLangOpts().OMPHostIRFile << EC.message();
3300 return;
3301 }
3302
3303 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
3304 if (!MD)
3305 return;
3306
3307 for (llvm::MDNode *MN : MD->operands()) {
3308 auto &&GetMDInt = [MN](unsigned Idx) {
3309 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3310 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3311 };
3312
3313 auto &&GetMDString = [MN](unsigned Idx) {
3314 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
3315 return V->getString();
3316 };
3317
3318 switch (GetMDInt(0)) {
3319 default:
3320 llvm_unreachable("Unexpected metadata!")::llvm::llvm_unreachable_internal("Unexpected metadata!", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3320)
;
3321 break;
3322 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3323 OffloadingEntryInfoTargetRegion:
3324 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3325 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
3326 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
3327 /*Order=*/GetMDInt(5));
3328 break;
3329 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3330 OffloadingEntryInfoDeviceGlobalVar:
3331 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
3332 /*MangledName=*/GetMDString(1),
3333 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3334 /*Flags=*/GetMDInt(2)),
3335 /*Order=*/GetMDInt(3));
3336 break;
3337 }
3338 }
3339}
3340
3341void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3342 if (!KmpRoutineEntryPtrTy) {
3343 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3344 ASTContext &C = CGM.getContext();
3345 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3346 FunctionProtoType::ExtProtoInfo EPI;
3347 KmpRoutineEntryPtrQTy = C.getPointerType(
3348 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3349 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3350 }
3351}
3352
3353QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
3354 // Make sure the type of the entry is already created. This is the type we
3355 // have to create:
3356 // struct __tgt_offload_entry{
3357 // void *addr; // Pointer to the offload entry info.
3358 // // (function or global)
3359 // char *name; // Name of the function or global.
3360 // size_t size; // Size of the entry info (0 if it a function).
3361 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
3362 // int32_t reserved; // Reserved, to use by the runtime library.
3363 // };
3364 if (TgtOffloadEntryQTy.isNull()) {
3365 ASTContext &C = CGM.getContext();
3366 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
3367 RD->startDefinition();
3368 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3369 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
3370 addFieldToRecordDecl(C, RD, C.getSizeType());
3371 addFieldToRecordDecl(
3372 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3373 addFieldToRecordDecl(
3374 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3375 RD->completeDefinition();
3376 RD->addAttr(PackedAttr::CreateImplicit(C));
3377 TgtOffloadEntryQTy = C.getRecordType(RD);
3378 }
3379 return TgtOffloadEntryQTy;
3380}
3381
3382namespace {
3383struct PrivateHelpersTy {
3384 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3385 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3386 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3387 PrivateElemInit(PrivateElemInit) {}
3388 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3389 const Expr *OriginalRef = nullptr;
3390 const VarDecl *Original = nullptr;
3391 const VarDecl *PrivateCopy = nullptr;
3392 const VarDecl *PrivateElemInit = nullptr;
3393 bool isLocalPrivate() const {
3394 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3395 }
3396};
3397typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3398} // anonymous namespace
3399
3400static bool isAllocatableDecl(const VarDecl *VD) {
3401 const VarDecl *CVD = VD->getCanonicalDecl();
3402 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3403 return false;
3404 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3405 // Use the default allocation.
3406 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
3407 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
3408 !AA->getAllocator());
3409}
3410
3411static RecordDecl *
3412createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3413 if (!Privates.empty()) {
3414 ASTContext &C = CGM.getContext();
3415 // Build struct .kmp_privates_t. {
3416 // /* private vars */
3417 // };
3418 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
3419 RD->startDefinition();
3420 for (const auto &Pair : Privates) {
3421 const VarDecl *VD = Pair.second.Original;
3422 QualType Type = VD->getType().getNonReferenceType();
3423 // If the private variable is a local variable with lvalue ref type,
3424 // allocate the pointer instead of the pointee type.
3425 if (Pair.second.isLocalPrivate()) {
3426 if (VD->getType()->isLValueReferenceType())
3427 Type = C.getPointerType(Type);
3428 if (isAllocatableDecl(VD))
3429 Type = C.getPointerType(Type);
3430 }
3431 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
3432 if (VD->hasAttrs()) {
3433 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3434 E(VD->getAttrs().end());
3435 I != E; ++I)
3436 FD->addAttr(*I);
3437 }
3438 }
3439 RD->completeDefinition();
3440 return RD;
3441 }
3442 return nullptr;
3443}
3444
3445static RecordDecl *
3446createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3447 QualType KmpInt32Ty,
3448 QualType KmpRoutineEntryPointerQTy) {
3449 ASTContext &C = CGM.getContext();
3450 // Build struct kmp_task_t {
3451 // void * shareds;
3452 // kmp_routine_entry_t routine;
3453 // kmp_int32 part_id;
3454 // kmp_cmplrdata_t data1;
3455 // kmp_cmplrdata_t data2;
3456 // For taskloops additional fields:
3457 // kmp_uint64 lb;
3458 // kmp_uint64 ub;
3459 // kmp_int64 st;
3460 // kmp_int32 liter;
3461 // void * reductions;
3462 // };
3463 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
3464 UD->startDefinition();
3465 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3466 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3467 UD->completeDefinition();
3468 QualType KmpCmplrdataTy = C.getRecordType(UD);
3469 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
3470 RD->startDefinition();
3471 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3472 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3473 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3474 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3475 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3476 if (isOpenMPTaskLoopDirective(Kind)) {
3477 QualType KmpUInt64Ty =
3478 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3479 QualType KmpInt64Ty =
3480 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3481 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3482 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3483 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3484 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3485 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3486 }
3487 RD->completeDefinition();
3488 return RD;
3489}
3490
3491static RecordDecl *
3492createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3493 ArrayRef<PrivateDataTy> Privates) {
3494 ASTContext &C = CGM.getContext();
3495 // Build struct kmp_task_t_with_privates {
3496 // kmp_task_t task_data;
3497 // .kmp_privates_t. privates;
3498 // };
3499 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3500 RD->startDefinition();
3501 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
3502 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3503 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
3504 RD->completeDefinition();
3505 return RD;
3506}
3507
3508/// Emit a proxy function which accepts kmp_task_t as the second
3509/// argument.
3510/// \code
3511/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3512/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3513/// For taskloops:
3514/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3515/// tt->reductions, tt->shareds);
3516/// return 0;
3517/// }
3518/// \endcode
3519static llvm::Function *
3520emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3521 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3522 QualType KmpTaskTWithPrivatesPtrQTy,
3523 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3524 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3525 llvm::Value *TaskPrivatesMap) {
3526 ASTContext &C = CGM.getContext();
3527 FunctionArgList Args;
3528 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3529 ImplicitParamDecl::Other);
3530 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3531 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3532 ImplicitParamDecl::Other);
3533 Args.push_back(&GtidArg);
3534 Args.push_back(&TaskTypeArg);
3535 const auto &TaskEntryFnInfo =
3536 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3537 llvm::FunctionType *TaskEntryTy =
3538 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3539 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
3540 auto *TaskEntry = llvm::Function::Create(
3541 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3542 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
3543 TaskEntry->setDoesNotRecurse();
3544 CodeGenFunction CGF(CGM);
3545 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3546 Loc, Loc);
3547
3548 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3549 // tt,
3550 // For taskloops:
3551 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3552 // tt->task_data.shareds);
3553 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3554 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3555 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3556 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3557 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3558 const auto *KmpTaskTWithPrivatesQTyRD =
3559 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3560 LValue Base =
3561 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3562 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
3563 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3564 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3565 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3566
3567 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3568 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3569 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3570 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3571 CGF.ConvertTypeForMem(SharedsPtrTy));
3572
3573 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3574 llvm::Value *PrivatesParam;
3575 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3576 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3577 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3578 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3579 } else {
3580 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3581 }
3582
3583 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3584 TaskPrivatesMap,
3585 CGF.Builder
3586 .CreatePointerBitCastOrAddrSpaceCast(
3587 TDBase.getAddress(CGF), CGF.VoidPtrTy)
3588 .getPointer()};
3589 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3590 std::end(CommonArgs));
3591 if (isOpenMPTaskLoopDirective(Kind)) {
3592 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3593 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3594 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3595 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3596 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3597 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3598 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3599 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3600 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3601 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3602 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3603 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3604 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3605 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3606 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3607 CallArgs.push_back(LBParam);
3608 CallArgs.push_back(UBParam);
3609 CallArgs.push_back(StParam);
3610 CallArgs.push_back(LIParam);
3611 CallArgs.push_back(RParam);
3612 }
3613 CallArgs.push_back(SharedsParam);
3614
3615 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3616 CallArgs);
3617 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3618 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3619 CGF.FinishFunction();
3620 return TaskEntry;
3621}
3622
3623static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3624 SourceLocation Loc,
3625 QualType KmpInt32Ty,
3626 QualType KmpTaskTWithPrivatesPtrQTy,
3627 QualType KmpTaskTWithPrivatesQTy) {
3628 ASTContext &C = CGM.getContext();
3629 FunctionArgList Args;
3630 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3631 ImplicitParamDecl::Other);
3632 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3633 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3634 ImplicitParamDecl::Other);
3635 Args.push_back(&GtidArg);
3636 Args.push_back(&TaskTypeArg);
3637 const auto &DestructorFnInfo =
3638 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3639 llvm::FunctionType *DestructorFnTy =
3640 CGM.getTypes().GetFunctionType(DestructorFnInfo);
3641 std::string Name =
3642 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
3643 auto *DestructorFn =
3644 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3645 Name, &CGM.getModule());
3646 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
3647 DestructorFnInfo);
3648 DestructorFn->setDoesNotRecurse();
3649 CodeGenFunction CGF(CGM);
3650 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3651 Args, Loc, Loc);
3652
3653 LValue Base = CGF.EmitLoadOfPointerLValue(
3654 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3655 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3656 const auto *KmpTaskTWithPrivatesQTyRD =
3657 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3658 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3659 Base = CGF.EmitLValueForField(Base, *FI);
3660 for (const auto *Field :
3661 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3662 if (QualType::DestructionKind DtorKind =
3663 Field->getType().isDestructedType()) {
3664 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3665 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
3666 }
3667 }
3668 CGF.FinishFunction();
3669 return DestructorFn;
3670}
3671
3672/// Emit a privates mapping function for correct handling of private and
3673/// firstprivate variables.
3674/// \code
3675/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3676/// **noalias priv1,..., <tyn> **noalias privn) {
3677/// *priv1 = &.privates.priv1;
3678/// ...;
3679/// *privn = &.privates.privn;
3680/// }
3681/// \endcode
3682static llvm::Value *
3683emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3684 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3685 ArrayRef<PrivateDataTy> Privates) {
3686 ASTContext &C = CGM.getContext();
3687 FunctionArgList Args;
3688 ImplicitParamDecl TaskPrivatesArg(
3689 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3690 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3691 ImplicitParamDecl::Other);
3692 Args.push_back(&TaskPrivatesArg);
3693 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3694 unsigned Counter = 1;
3695 for (const Expr *E : Data.PrivateVars) {
3696 Args.push_back(ImplicitParamDecl::Create(
3697 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3698 C.getPointerType(C.getPointerType(E->getType()))
3699 .withConst()
3700 .withRestrict(),
3701 ImplicitParamDecl::Other));
3702 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3703 PrivateVarsPos[VD] = Counter;
3704 ++Counter;
3705 }
3706 for (const Expr *E : Data.FirstprivateVars) {
3707 Args.push_back(ImplicitParamDecl::Create(
3708 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3709 C.getPointerType(C.getPointerType(E->getType()))
3710 .withConst()
3711 .withRestrict(),
3712 ImplicitParamDecl::Other));
3713 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3714 PrivateVarsPos[VD] = Counter;
3715 ++Counter;
3716 }
3717 for (const Expr *E : Data.LastprivateVars) {
3718 Args.push_back(ImplicitParamDecl::Create(
3719 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3720 C.getPointerType(C.getPointerType(E->getType()))
3721 .withConst()
3722 .withRestrict(),
3723 ImplicitParamDecl::Other));
3724 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3725 PrivateVarsPos[VD] = Counter;
3726 ++Counter;
3727 }
3728 for (const VarDecl *VD : Data.PrivateLocals) {
3729 QualType Ty = VD->getType().getNonReferenceType();
3730 if (VD->getType()->isLValueReferenceType())
3731 Ty = C.getPointerType(Ty);
3732 if (isAllocatableDecl(VD))
3733 Ty = C.getPointerType(Ty);
3734 Args.push_back(ImplicitParamDecl::Create(
3735 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3736 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3737 ImplicitParamDecl::Other));
3738 PrivateVarsPos[VD] = Counter;
3739 ++Counter;
3740 }
3741 const auto &TaskPrivatesMapFnInfo =
3742 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3743 llvm::FunctionType *TaskPrivatesMapTy =
3744 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
3745 std::string Name =
3746 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
3747 auto *TaskPrivatesMap = llvm::Function::Create(
3748 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3749 &CGM.getModule());
3750 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
3751 TaskPrivatesMapFnInfo);
3752 if (CGM.getLangOpts().Optimize) {
3753 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3754 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3755 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3756 }
3757 CodeGenFunction CGF(CGM);
3758 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3759 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3760
3761 // *privi = &.privates.privi;
3762 LValue Base = CGF.EmitLoadOfPointerLValue(
3763 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3764 TaskPrivatesArg.getType()->castAs<PointerType>());
3765 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
3766 Counter = 0;
3767 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3768 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3769 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3770 LValue RefLVal =
3771 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3772 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3773 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
3774 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3775 ++Counter;
3776 }
3777 CGF.FinishFunction();
3778 return TaskPrivatesMap;
3779}
3780
3781/// Emit initialization for private variables in task-based directives.
3782static void emitPrivatesInit(CodeGenFunction &CGF,
3783 const OMPExecutableDirective &D,
3784 Address KmpTaskSharedsPtr, LValue TDBase,
3785 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3786 QualType SharedsTy, QualType SharedsPtrTy,
3787 const OMPTaskDataTy &Data,
3788 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3789 ASTContext &C = CGF.getContext();
3790 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3791 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3792 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
3793 ? OMPD_taskloop
3794 : OMPD_task;
3795 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
3796 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3797 LValue SrcBase;
3798 bool IsTargetTask =
3799 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
3800 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
3801 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3802 // PointersArray, SizesArray, and MappersArray. The original variables for
3803 // these arrays are not captured and we get their addresses explicitly.
3804 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3805 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3806 SrcBase = CGF.MakeAddrLValue(
3807 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3808 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
3809 SharedsTy);
3810 }
3811 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3812 for (const PrivateDataTy &Pair : Privates) {
3813 // Do not initialize private locals.
3814 if (Pair.second.isLocalPrivate()) {
3815 ++FI;
3816 continue;
3817 }
3818 const VarDecl *VD = Pair.second.PrivateCopy;
3819 const Expr *Init = VD->getAnyInitializer();
3820 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3821 !CGF.isTrivialInitializer(Init)))) {
3822 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3823 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3824 const VarDecl *OriginalVD = Pair.second.Original;
3825 // Check if the variable is the target-based BasePointersArray,
3826 // PointersArray, SizesArray, or MappersArray.
3827 LValue SharedRefLValue;
3828 QualType Type = PrivateLValue.getType();
3829 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3830 if (IsTargetTask && !SharedField) {
3831 assert(isa<ImplicitParamDecl>(OriginalVD) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3832 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3833 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3834 ->getNumParams() == 0 &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3835 isa<TranslationUnitDecl>(((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3836 cast<CapturedDecl>(OriginalVD->getDeclContext())((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3837 ->getDeclContext()) &&((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
3838 "Expected artificial target data variable.")((isa<ImplicitParamDecl>(OriginalVD) && isa<
CapturedDecl>(OriginalVD->getDeclContext()) && cast
<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams
() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl
>(OriginalVD->getDeclContext()) ->getDeclContext()) &&
"Expected artificial target data variable.") ? static_cast<
void> (0) : __assert_fail ("isa<ImplicitParamDecl>(OriginalVD) && isa<CapturedDecl>(OriginalVD->getDeclContext()) && cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa<TranslationUnitDecl>( cast<CapturedDecl>(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3838, __PRETTY_FUNCTION__))
;
3839 SharedRefLValue =
3840 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3841 } else if (ForDup) {
3842 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3843 SharedRefLValue = CGF.MakeAddrLValue(
3844 Address(SharedRefLValue.getPointer(CGF),
3845 C.getDeclAlign(OriginalVD)),
3846 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3847 SharedRefLValue.getTBAAInfo());
3848 } else if (CGF.LambdaCaptureFields.count(
3849 Pair.second.Original->getCanonicalDecl()) > 0 ||
3850 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) {
3851 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3852 } else {
3853 // Processing for implicitly captured variables.
3854 InlinedOpenMPRegionRAII Region(
3855 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3856 /*HasCancel=*/false);
3857 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3858 }
3859 if (Type->isArrayType()) {
3860 // Initialize firstprivate array.
3861 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3862 // Perform simple memcpy.
3863 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3864 } else {
3865 // Initialize firstprivate array using element-by-element
3866 // initialization.
3867 CGF.EmitOMPAggregateAssign(
3868 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
3869 Type,
3870 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3871 Address SrcElement) {
3872 // Clean up any temporaries needed by the initialization.
3873 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3874 InitScope.addPrivate(
3875 Elem, [SrcElement]() -> Address { return SrcElement; });
3876 (void)InitScope.Privatize();
3877 // Emit initialization for single element.
3878 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3879 CGF, &CapturesInfo);
3880 CGF.EmitAnyExprToMem(Init, DestElement,
3881 Init->getType().getQualifiers(),
3882 /*IsInitializer=*/false);
3883 });
3884 }
3885 } else {
3886 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3887 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
3888 return SharedRefLValue.getAddress(CGF);
3889 });
3890 (void)InitScope.Privatize();
3891 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3892 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3893 /*capturedByInit=*/false);
3894 }
3895 } else {
3896 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3897 }
3898 }
3899 ++FI;
3900 }
3901}
3902
3903/// Check if duplication function is required for taskloops.
3904static bool checkInitIsRequired(CodeGenFunction &CGF,
3905 ArrayRef<PrivateDataTy> Privates) {
3906 bool InitRequired = false;
3907 for (const PrivateDataTy &Pair : Privates) {
3908 if (Pair.second.isLocalPrivate())
3909 continue;
3910 const VarDecl *VD = Pair.second.PrivateCopy;
3911 const Expr *Init = VD->getAnyInitializer();
3912 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
3913 !CGF.isTrivialInitializer(Init));
3914 if (InitRequired)
3915 break;
3916 }
3917 return InitRequired;
3918}
3919
3920
3921/// Emit task_dup function (for initialization of
3922/// private/firstprivate/lastprivate vars and last_iter flag)
3923/// \code
3924/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3925/// lastpriv) {
3926/// // setup lastprivate flag
3927/// task_dst->last = lastpriv;
3928/// // could be constructor calls here...
3929/// }
3930/// \endcode
3931static llvm::Value *
3932emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3933 const OMPExecutableDirective &D,
3934 QualType KmpTaskTWithPrivatesPtrQTy,
3935 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3936 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3937 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3938 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3939 ASTContext &C = CGM.getContext();
3940 FunctionArgList Args;
3941 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3942 KmpTaskTWithPrivatesPtrQTy,
3943 ImplicitParamDecl::Other);
3944 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3945 KmpTaskTWithPrivatesPtrQTy,
3946 ImplicitParamDecl::Other);
3947 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
3948 ImplicitParamDecl::Other);
3949 Args.push_back(&DstArg);
3950 Args.push_back(&SrcArg);
3951 Args.push_back(&LastprivArg);
3952 const auto &TaskDupFnInfo =
3953 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3954 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
3955 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
3956 auto *TaskDup = llvm::Function::Create(
3957 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3958 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
3959 TaskDup->setDoesNotRecurse();
3960 CodeGenFunction CGF(CGM);
3961 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3962 Loc);
3963
3964 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3965 CGF.GetAddrOfLocalVar(&DstArg),
3966 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3967 // task_dst->liter = lastpriv;
3968 if (WithLastIter) {
3969 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3970 LValue Base = CGF.EmitLValueForField(
3971 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3972 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3973 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3974 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3975 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3976 }
3977
3978 // Emit initial values for private copies (if any).
3979 assert(!Privates.empty())((!Privates.empty()) ? static_cast<void> (0) : __assert_fail
("!Privates.empty()", "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 3979, __PRETTY_FUNCTION__))
;
3980 Address KmpTaskSharedsPtr = Address::invalid();
3981 if (!Data.FirstprivateVars.empty()) {
3982 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3983 CGF.GetAddrOfLocalVar(&SrcArg),
3984 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3985 LValue Base = CGF.EmitLValueForField(
3986 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3987 KmpTaskSharedsPtr = Address(
3988 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3989 Base, *std::next(KmpTaskTQTyRD->field_begin(),
3990 KmpTaskTShareds)),
3991 Loc),
3992 CGM.getNaturalTypeAlignment(SharedsTy));
3993 }
3994 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3995 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3996 CGF.FinishFunction();
3997 return TaskDup;
3998}
3999
4000/// Checks if destructor function is required to be generated.
4001/// \return true if cleanups are required, false otherwise.
4002static bool
4003checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4004 ArrayRef<PrivateDataTy> Privates) {
4005 for (const PrivateDataTy &P : Privates) {
4006 if (P.second.isLocalPrivate())
4007 continue;
4008 QualType Ty = P.second.Original->getType().getNonReferenceType();
4009 if (Ty.isDestructedType())
4010 return true;
4011 }
4012 return false;
4013}
4014
4015namespace {
4016/// Loop generator for OpenMP iterator expression.
4017class OMPIteratorGeneratorScope final
4018 : public CodeGenFunction::OMPPrivateScope {
4019 CodeGenFunction &CGF;
4020 const OMPIteratorExpr *E = nullptr;
4021 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
4022 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
4023 OMPIteratorGeneratorScope() = delete;
4024 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
4025
4026public:
4027 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
4028 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
4029 if (!E)
4030 return;
4031 SmallVector<llvm::Value *, 4> Uppers;
4032 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4033 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
4034 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
4035 addPrivate(VD, [&CGF, VD]() {
4036 return CGF.CreateMemTemp(VD->getType(), VD->getName());
4037 });
4038 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4039 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() {
4040 return CGF.CreateMemTemp(HelperData.CounterVD->getType(),
4041 "counter.addr");
4042 });
4043 }
4044 Privatize();
4045
4046 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
4047 const OMPIteratorHelperData &HelperData = E->getHelper(I);
4048 LValue CLVal =
4049 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
4050 HelperData.CounterVD->getType());
4051 // Counter = 0;
4052 CGF.EmitStoreOfScalar(
4053 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0),
4054 CLVal);
4055 CodeGenFunction::JumpDest &ContDest =
4056 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
4057 CodeGenFunction::JumpDest &ExitDest =
4058 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
4059 // N = <number-of_iterations>;
4060 llvm::Value *N = Uppers[I];
4061 // cont:
4062 // if (Counter < N) goto body; else goto exit;
4063 CGF.EmitBlock(ContDest.getBlock());
4064 auto *CVal =
4065 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
4066 llvm::Value *Cmp =
4067 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
4068 ? CGF.Builder.CreateICmpSLT(CVal, N)
4069 : CGF.Builder.CreateICmpULT(CVal, N);
4070 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
4071 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
4072 // body:
4073 CGF.EmitBlock(BodyBB);
4074 // Iteri = Begini + Counter * Stepi;
4075 CGF.EmitIgnoredExpr(HelperData.Update);
4076 }
4077 }
4078 ~OMPIteratorGeneratorScope() {
4079 if (!E)
4080 return;
4081 for (unsigned I = E->numOfIterators(); I > 0; --I) {
4082 // Counter = Counter + 1;
4083 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);
4084 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
4085 // goto cont;
4086 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
4087 // exit:
4088 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
4089 }
4090 }
4091};
4092} // namespace
4093
4094static std::pair<llvm::Value *, llvm::Value *>
4095getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
4096 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
4097 llvm::Value *Addr;
4098 if (OASE) {
4099 const Expr *Base = OASE->getBase();
4100 Addr = CGF.EmitScalarExpr(Base);
4101 } else {
4102 Addr = CGF.EmitLValue(E).getPointer(CGF);
4103 }
4104 llvm::Value *SizeVal;
4105 QualType Ty = E->getType();
4106 if (OASE) {
4107 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
4108 for (const Expr *SE : OASE->getDimensions()) {
4109 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
4110 Sz = CGF.EmitScalarConversion(
4111 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
4112 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
4113 }
4114 } else if (const auto *ASE =
4115 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
4116 LValue UpAddrLVal =
4117 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
4118 llvm::Value *UpAddr =
4119 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
4120 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
4121 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
4122 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4123 } else {
4124 SizeVal = CGF.getTypeSize(Ty);
4125 }
4126 return std::make_pair(Addr, SizeVal);
4127}
4128
4129/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4130static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
4131 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);
4132 if (KmpTaskAffinityInfoTy.isNull()) {
4133 RecordDecl *KmpAffinityInfoRD =
4134 C.buildImplicitRecord("kmp_task_affinity_info_t");
4135 KmpAffinityInfoRD->startDefinition();
4136 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());
4137 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
4138 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
4139 KmpAffinityInfoRD->completeDefinition();
4140 KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
4141 }
4142}
4143
4144CGOpenMPRuntime::TaskResultTy
4145CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4146 const OMPExecutableDirective &D,
4147 llvm::Function *TaskFunction, QualType SharedsTy,
4148 Address Shareds, const OMPTaskDataTy &Data) {
4149 ASTContext &C = CGM.getContext();
4150 llvm::SmallVector<PrivateDataTy, 4> Privates;
4151 // Aggregate privates and sort them by the alignment.
4152 const auto *I = Data.PrivateCopies.begin();
4153 for (const Expr *E : Data.PrivateVars) {
4154 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4155 Privates.emplace_back(
4156 C.getDeclAlign(VD),
4157 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4158 /*PrivateElemInit=*/nullptr));
4159 ++I;
4160 }
4161 I = Data.FirstprivateCopies.begin();
4162 const auto *IElemInitRef = Data.FirstprivateInits.begin();
4163 for (const Expr *E : Data.FirstprivateVars) {
4164 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4165 Privates.emplace_back(
4166 C.getDeclAlign(VD),
4167 PrivateHelpersTy(
4168 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4169 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4170 ++I;
4171 ++IElemInitRef;
4172 }
4173 I = Data.LastprivateCopies.begin();
4174 for (const Expr *E : Data.LastprivateVars) {
4175 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4176 Privates.emplace_back(
4177 C.getDeclAlign(VD),
4178 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4179 /*PrivateElemInit=*/nullptr));
4180 ++I;
4181 }
4182 for (const VarDecl *VD : Data.PrivateLocals) {
4183 if (isAllocatableDecl(VD))
4184 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));
4185 else
4186 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));
4187 }
4188 llvm::stable_sort(Privates,
4189 [](const PrivateDataTy &L, const PrivateDataTy &R) {
4190 return L.first > R.first;
4191 });
4192 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4193 // Build type kmp_routine_entry_t (if not built yet).
4194 emitKmpRoutineEntryT(KmpInt32Ty);
4195 // Build type kmp_task_t (if not built yet).
4196 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4197 if (SavedKmpTaskloopTQTy.isNull()) {
4198 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4199 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4200 }
4201 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4202 } else {
4203 assert((D.getDirectiveKind() == OMPD_task ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4206, __PRETTY_FUNCTION__))
4204 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4206, __PRETTY_FUNCTION__))
4205 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4206, __PRETTY_FUNCTION__))
4206 "Expected taskloop, task or target directive")(((D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective
(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective
(D.getDirectiveKind())) && "Expected taskloop, task or target directive"
) ? static_cast<void> (0) : __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4206, __PRETTY_FUNCTION__))
;
4207 if (SavedKmpTaskTQTy.isNull()) {
4208 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4209 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4210 }
4211 KmpTaskTQTy = SavedKmpTaskTQTy;
4212 }
4213 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4214 // Build particular struct kmp_task_t for the given task.
4215 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4216 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4217 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4218 QualType KmpTaskTWithPrivatesPtrQTy =
4219 C.getPointerType(KmpTaskTWithPrivatesQTy);
4220 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4221 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4222 KmpTaskTWithPrivatesTy->getPointerTo();
4223 llvm::Value *KmpTaskTWithPrivatesTySize =
4224 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4225 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4226
4227 // Emit initial values for private copies (if any).
4228 llvm::Value *TaskPrivatesMap = nullptr;
4229 llvm::Type *TaskPrivatesMapTy =
4230 std::next(TaskFunction->arg_begin(), 3)->getType();
4231 if (!Privates.empty()) {
4232 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4233 TaskPrivatesMap =
4234 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);
4235 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4236 TaskPrivatesMap, TaskPrivatesMapTy);
4237 } else {
4238 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4239 cast<llvm::PointerType>(TaskPrivatesMapTy));
4240 }
4241 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4242 // kmp_task_t *tt);
4243 llvm::Function *TaskEntry = emitProxyTaskFunction(
4244 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4245 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4246 TaskPrivatesMap);
4247
4248 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4249 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4250 // kmp_routine_entry_t *task_entry);
4251 // Task flags. Format is taken from
4252 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
4253 // description of kmp_tasking_flags struct.
4254 enum {
4255 TiedFlag = 0x1,
4256 FinalFlag = 0x2,
4257 DestructorsFlag = 0x8,
4258 PriorityFlag = 0x20,
4259 DetachableFlag = 0x40,
4260 };
4261 unsigned Flags = Data.Tied ? TiedFlag : 0;
4262 bool NeedsCleanup = false;
4263 if (!Privates.empty()) {
4264 NeedsCleanup =
4265 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4266 if (NeedsCleanup)
4267 Flags = Flags | DestructorsFlag;
4268 }
4269 if (Data.Priority.getInt())
4270 Flags = Flags | PriorityFlag;
4271 if (D.hasClausesOfKind<OMPDetachClause>())
4272 Flags = Flags | DetachableFlag;
4273 llvm::Value *TaskFlags =
4274 Data.Final.getPointer()
4275 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4276 CGF.Builder.getInt32(FinalFlag),
4277 CGF.Builder.getInt32(/*C=*/0))
4278 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4279 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4280 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4281 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4282 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4283 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4284 TaskEntry, KmpRoutineEntryPtrTy)};
4285 llvm::Value *NewTask;
4286 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4287 // Check if we have any device clause associated with the directive.
4288 const Expr *Device = nullptr;
4289 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4290 Device = C->getDevice();
4291 // Emit device ID if any otherwise use default value.
4292 llvm::Value *DeviceID;
4293 if (Device)
4294 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
4295 CGF.Int64Ty, /*isSigned=*/true);
4296 else
4297 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
4298 AllocArgs.push_back(DeviceID);
4299 NewTask = CGF.EmitRuntimeCall(
4300 OMPBuilder.getOrCreateRuntimeFunction(
4301 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4302 AllocArgs);
4303 } else {
4304 NewTask =
4305 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4306 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4307 AllocArgs);
4308 }
4309 // Emit detach clause initialization.
4310 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4311 // task_descriptor);
4312 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4313 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4314 LValue EvtLVal = CGF.EmitLValue(Evt);
4315
4316 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4317 // int gtid, kmp_task_t *task);
4318 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
4319 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
4320 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
4321 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4322 OMPBuilder.getOrCreateRuntimeFunction(
4323 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4324 {Loc, Tid, NewTask});
4325 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
4326 Evt->getExprLoc());
4327 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
4328 }
4329 // Process affinity clauses.
4330 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4331 // Process list of affinity data.
4332 ASTContext &C = CGM.getContext();
4333 Address AffinitiesArray = Address::invalid();
4334 // Calculate number of elements to form the array of affinity data.
4335 llvm::Value *NumOfElements = nullptr;
4336 unsigned NumAffinities = 0;
4337 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4338 if (const Expr *Modifier = C->getModifier()) {
4339 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4340 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4341 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4342 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4343 NumOfElements =
4344 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4345 }
4346 } else {
4347 NumAffinities += C->varlist_size();
4348 }
4349 }
4350 getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy);
4351 // Fields ids in kmp_task_affinity_info record.
4352 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4353
4354 QualType KmpTaskAffinityInfoArrayTy;
4355 if (NumOfElements) {
4356 NumOfElements = CGF.Builder.CreateNUWAdd(
4357 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
4358 OpaqueValueExpr OVE(
4359 Loc,
4360 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),
4361 VK_RValue);
4362 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4363 RValue::get(NumOfElements));
4364 KmpTaskAffinityInfoArrayTy =
4365 C.getVariableArrayType(KmpTaskAffinityInfoTy, &OVE, ArrayType::Normal,
4366 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4367 // Properly emit variable-sized array.
4368 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,
4369 ImplicitParamDecl::Other);
4370 CGF.EmitVarDecl(*PD);
4371 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
4372 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4373 /*isSigned=*/false);
4374 } else {
4375 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4376 KmpTaskAffinityInfoTy,
4377 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,
4378 ArrayType::Normal, /*IndexTypeQuals=*/0);
4379 AffinitiesArray =
4380 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");
4381 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
4382 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,
4383 /*isSigned=*/false);
4384 }
4385
4386 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4387 // Fill array by elements without iterators.
4388 unsigned Pos = 0;
4389 bool HasIterator = false;
4390 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4391 if (C->getModifier()) {
4392 HasIterator = true;
4393 continue;
4394 }
4395 for (const Expr *E : C->varlists()) {
4396 llvm::Value *Addr;
4397 llvm::Value *Size;
4398 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4399 LValue Base =
4400 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
4401 KmpTaskAffinityInfoTy);
4402 // affs[i].base_addr = &<Affinities[i].second>;
4403 LValue BaseAddrLVal = CGF.EmitLValueForField(
4404 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4405 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4406 BaseAddrLVal);
4407 // affs[i].len = sizeof(<Affinities[i].second>);
4408 LValue LenLVal = CGF.EmitLValueForField(
4409 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4410 CGF.EmitStoreOfScalar(Size, LenLVal);
4411 ++Pos;
4412 }
4413 }
4414 LValue PosLVal;
4415 if (HasIterator) {
4416 PosLVal = CGF.MakeAddrLValue(
4417 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),
4418 C.getSizeType());
4419 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4420 }
4421 // Process elements with iterators.
4422 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4423 const Expr *Modifier = C->getModifier();
4424 if (!Modifier)
4425 continue;
4426 OMPIteratorGeneratorScope IteratorScope(
4427 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
4428 for (const Expr *E : C->varlists()) {
4429 llvm::Value *Addr;
4430 llvm::Value *Size;
4431 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4432 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4433 LValue Base = CGF.MakeAddrLValue(
4434 Address(CGF.Builder.CreateGEP(AffinitiesArray.getPointer(), Idx),
4435 AffinitiesArray.getAlignment()),
4436 KmpTaskAffinityInfoTy);
4437 // affs[i].base_addr = &<Affinities[i].second>;
4438 LValue BaseAddrLVal = CGF.EmitLValueForField(
4439 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4440 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4441 BaseAddrLVal);
4442 // affs[i].len = sizeof(<Affinities[i].second>);
4443 LValue LenLVal = CGF.EmitLValueForField(
4444 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4445 CGF.EmitStoreOfScalar(Size, LenLVal);
4446 Idx = CGF.Builder.CreateNUWAdd(
4447 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4448 CGF.EmitStoreOfScalar(Idx, PosLVal);
4449 }
4450 }
4451 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4452 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4453 // naffins, kmp_task_affinity_info_t *affin_list);
4454 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4455 llvm::Value *GTid = getThreadID(CGF, Loc);
4456 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4457 AffinitiesArray.getPointer(), CGM.VoidPtrTy);
4458 // FIXME: Emit the function and ignore its result for now unless the
4459 // runtime function is properly implemented.
4460 (void)CGF.EmitRuntimeCall(
4461 OMPBuilder.getOrCreateRuntimeFunction(
4462 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4463 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4464 }
4465 llvm::Value *NewTaskNewTaskTTy =
4466 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4467 NewTask, KmpTaskTWithPrivatesPtrTy);
4468 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4469 KmpTaskTWithPrivatesQTy);
4470 LValue TDBase =
4471 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4472 // Fill the data in the resulting kmp_task_t record.
4473 // Copy shareds if there are any.
4474 Address KmpTaskSharedsPtr = Address::invalid();
4475 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4476 KmpTaskSharedsPtr =
4477 Address(CGF.EmitLoadOfScalar(
4478 CGF.EmitLValueForField(
4479 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4480 KmpTaskTShareds)),
4481 Loc),
4482 CGM.getNaturalTypeAlignment(SharedsTy));
4483 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4484 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4485 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4486 }
4487 // Emit initial values for private copies (if any).
4488 TaskResultTy Result;
4489 if (!Privates.empty()) {
4490 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4491 SharedsTy, SharedsPtrTy, Data, Privates,
4492 /*ForDup=*/false);
4493 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4494 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4495 Result.TaskDupFn = emitTaskDupFunction(
4496 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4497 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4498 /*WithLastIter=*/!Data.LastprivateVars.empty());
4499 }
4500 }
4501 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4502 enum { Priority = 0, Destructors = 1 };
4503 // Provide pointer to function with destructors for privates.
4504 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4505 const RecordDecl *KmpCmplrdataUD =
4506 (*FI)->getType()->getAsUnionType()->getDecl();
4507 if (NeedsCleanup) {
4508 llvm::Value *DestructorFn = emitDestructorsFunction(
4509 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4510 KmpTaskTWithPrivatesQTy);
4511 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4512 LValue DestructorsLV = CGF.EmitLValueForField(
4513 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4514 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4515 DestructorFn, KmpRoutineEntryPtrTy),
4516 DestructorsLV);
4517 }
4518 // Set priority.
4519 if (Data.Priority.getInt()) {
4520 LValue Data2LV = CGF.EmitLValueForField(
4521 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4522 LValue PriorityLV = CGF.EmitLValueForField(
4523 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4524 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4525 }
4526 Result.NewTask = NewTask;
4527 Result.TaskEntry = TaskEntry;
4528 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4529 Result.TDBase = TDBase;
4530 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4531 return Result;
4532}
4533
4534namespace {
4535/// Dependence kind for RTL.
4536enum RTLDependenceKindTy {
4537 DepIn = 0x01,
4538 DepInOut = 0x3,
4539 DepMutexInOutSet = 0x4
4540};
4541/// Fields ids in kmp_depend_info record.
4542enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4543} // namespace
4544
4545/// Translates internal dependency kind into the runtime kind.
4546static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4547 RTLDependenceKindTy DepKind;
4548 switch (K) {
4549 case OMPC_DEPEND_in:
4550 DepKind = DepIn;
4551 break;
4552 // Out and InOut dependencies must use the same code.
4553 case OMPC_DEPEND_out:
4554 case OMPC_DEPEND_inout:
4555 DepKind = DepInOut;
4556 break;
4557 case OMPC_DEPEND_mutexinoutset:
4558 DepKind = DepMutexInOutSet;
4559 break;
4560 case OMPC_DEPEND_source:
4561 case OMPC_DEPEND_sink:
4562 case OMPC_DEPEND_depobj:
4563 case OMPC_DEPEND_unknown:
4564 llvm_unreachable("Unknown task dependence type")::llvm::llvm_unreachable_internal("Unknown task dependence type"
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4564)
;
4565 }
4566 return DepKind;
4567}
4568
4569/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4570static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4571 QualType &FlagsTy) {
4572 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4573 if (KmpDependInfoTy.isNull()) {
4574 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4575 KmpDependInfoRD->startDefinition();
4576 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4577 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4578 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4579 KmpDependInfoRD->completeDefinition();
4580 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
4581 }
4582}
4583
4584std::pair<llvm::Value *, LValue>
4585CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4586 SourceLocation Loc) {
4587 ASTContext &C = CGM.getContext();
4588 QualType FlagsTy;
4589 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4590 RecordDecl *KmpDependInfoRD =
4591 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4592 LValue Base = CGF.EmitLoadOfPointerLValue(
4593 DepobjLVal.getAddress(CGF),
4594 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4595 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4596 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4597 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
4598 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4599 Base.getTBAAInfo());
4600 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4601 Addr.getPointer(),
4602 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4603 LValue NumDepsBase = CGF.MakeAddrLValue(
4604 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4605 Base.getBaseInfo(), Base.getTBAAInfo());
4606 // NumDeps = deps[i].base_addr;
4607 LValue BaseAddrLVal = CGF.EmitLValueForField(
4608 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4609 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4610 return std::make_pair(NumDeps, Base);
4611}
4612
4613static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4614 llvm::PointerUnion<unsigned *, LValue *> Pos,
4615 const OMPTaskDataTy::DependData &Data,
4616 Address DependenciesArray) {
4617 CodeGenModule &CGM = CGF.CGM;
4618 ASTContext &C = CGM.getContext();
4619 QualType FlagsTy;
4620 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4621 RecordDecl *KmpDependInfoRD =
4622 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4623 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4624
4625 OMPIteratorGeneratorScope IteratorScope(
4626 CGF, cast_or_null<OMPIteratorExpr>(
4627 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4628 : nullptr));
4629 for (const Expr *E : Data.DepExprs) {
4630 llvm::Value *Addr;
4631 llvm::Value *Size;
4632 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4633 LValue Base;
4634 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4635 Base = CGF.MakeAddrLValue(
4636 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4637 } else {
4638 LValue &PosLVal = *Pos.get<LValue *>();
4639 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4640 Base = CGF.MakeAddrLValue(
4641 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Idx),
4642 DependenciesArray.getAlignment()),
4643 KmpDependInfoTy);
4644 }
4645 // deps[i].base_addr = &<Dependencies[i].second>;
4646 LValue BaseAddrLVal = CGF.EmitLValueForField(
4647 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4648 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4649 BaseAddrLVal);
4650 // deps[i].len = sizeof(<Dependencies[i].second>);
4651 LValue LenLVal = CGF.EmitLValueForField(
4652 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
4653 CGF.EmitStoreOfScalar(Size, LenLVal);
4654 // deps[i].flags = <Dependencies[i].first>;
4655 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);
4656 LValue FlagsLVal = CGF.EmitLValueForField(
4657 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
4658 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
4659 FlagsLVal);
4660 if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
4661 ++(*P);
4662 } else {
4663 LValue &PosLVal = *Pos.get<LValue *>();
4664 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4665 Idx = CGF.Builder.CreateNUWAdd(Idx,
4666 llvm::ConstantInt::get(Idx->getType(), 1));
4667 CGF.EmitStoreOfScalar(Idx, PosLVal);
4668 }
4669 }
4670}
4671
4672static SmallVector<llvm::Value *, 4>
4673emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4674 const OMPTaskDataTy::DependData &Data) {
4675 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4676, __PRETTY_FUNCTION__))
4676 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4676, __PRETTY_FUNCTION__))
;
4677 SmallVector<llvm::Value *, 4> Sizes;
4678 SmallVector<LValue, 4> SizeLVals;
4679 ASTContext &C = CGF.getContext();
4680 QualType FlagsTy;
4681 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4682 RecordDecl *KmpDependInfoRD =
4683 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4684 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4685 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4686 {
4687 OMPIteratorGeneratorScope IteratorScope(
4688 CGF, cast_or_null<OMPIteratorExpr>(
4689 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4690 : nullptr));
4691 for (const Expr *E : Data.DepExprs) {
4692 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4693 LValue Base = CGF.EmitLoadOfPointerLValue(
4694 DepobjLVal.getAddress(CGF),
4695 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4696 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4697 Base.getAddress(CGF), KmpDependInfoPtrT);
4698 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4699 Base.getTBAAInfo());
4700 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4701 Addr.getPointer(),
4702 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4703 LValue NumDepsBase = CGF.MakeAddrLValue(
4704 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4705 Base.getBaseInfo(), Base.getTBAAInfo());
4706 // NumDeps = deps[i].base_addr;
4707 LValue BaseAddrLVal = CGF.EmitLValueForField(
4708 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4709 llvm::Value *NumDeps =
4710 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4711 LValue NumLVal = CGF.MakeAddrLValue(
4712 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
4713 C.getUIntPtrType());
4714 CGF.InitTempAlloca(NumLVal.getAddress(CGF),
4715 llvm::ConstantInt::get(CGF.IntPtrTy, 0));
4716 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4717 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4718 CGF.EmitStoreOfScalar(Add, NumLVal);
4719 SizeLVals.push_back(NumLVal);
4720 }
4721 }
4722 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4723 llvm::Value *Size =
4724 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4725 Sizes.push_back(Size);
4726 }
4727 return Sizes;
4728}
4729
4730static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4731 LValue PosLVal,
4732 const OMPTaskDataTy::DependData &Data,
4733 Address DependenciesArray) {
4734 assert(Data.DepKind == OMPC_DEPEND_depobj &&((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4735, __PRETTY_FUNCTION__))
4735 "Expected depobj dependecy kind.")((Data.DepKind == OMPC_DEPEND_depobj && "Expected depobj dependecy kind."
) ? static_cast<void> (0) : __assert_fail ("Data.DepKind == OMPC_DEPEND_depobj && \"Expected depobj dependecy kind.\""
, "/build/llvm-toolchain-snapshot-13~++20210301100612+564f5b0734bd/clang/lib/CodeGen/CGOpenMPRuntime.cpp"
, 4735, __PRETTY_FUNCTION__))
;
4736 ASTContext &C = CGF.getContext();
4737 QualType FlagsTy;
4738 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4739 RecordDecl *KmpDependInfoRD =
4740 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4741 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4742 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
4743 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4744 {
4745 OMPIteratorGeneratorScope IteratorScope(
4746 CGF, cast_or_null<OMPIteratorExpr>(
4747 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4748 : nullptr));
4749 for (unsigned I = 0, End = Data.DepExprs.size(); I < End; ++I) {
4750 const Expr *E = Data.DepExprs[I];
4751 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4752 LValue Base = CGF.EmitLoadOfPointerLValue(
4753 DepobjLVal.getAddress(CGF),
4754 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
4755 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4756 Base.getAddress(CGF), KmpDependInfoPtrT);
4757 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
4758 Base.getTBAAInfo());
4759
4760 // Get number of elements in a single depobj.
4761 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4762 Addr.getPointer(),
4763 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4764 LValue NumDepsBase = CGF.MakeAddrLValue(
4765 Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
4766 Base.getBaseInfo(), Base.getTBAAInfo());
4767 // NumDeps = deps[i].base_addr;
4768 LValue BaseAddrLVal = CGF.EmitLValueForField(
4769 NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4770 llvm::Value *NumDeps =
4771 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
4772
4773 // memcopy dependency data.
4774 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4775 ElSize,
4776 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4777 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4778 Address DepAddr =
4779 Address(CGF.Builder.CreateGEP(DependenciesArray.getPointer(), Pos),
4780 DependenciesArray.getAlignment());
4781 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
4782
4783 // Increase pos.
4784 // pos += size;
4785 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4786 CGF.EmitStoreOfScalar(Add, PosLVal);
4787 }
4788 }
4789}
4790
4791std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4792 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4793 SourceLocation Loc) {
4794 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {
4795 return D.DepExprs.empty();
4796 }))
4797 return std::make_pair(nullptr, Address::invalid());
4798 // Process list of dependencies.
4799 ASTContext &C = CGM.getContext();
4800 Address DependenciesArray = Address::invalid();
4801 llvm::Value *NumOfElements = nullptr;
4802 unsigned NumDependencies = std::accumulate(
4803 Dependencies.begin(), Dependencies.end(), 0,
4804 [](unsigned V, const OMPTaskDataTy::DependData &D) {
4805 return D.DepKind == OMPC_DEPEND_depobj
4806 ? V
4807 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4808 });
4809 QualType FlagsTy;
4810 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4811 bool HasDepobjDeps = false;
4812 bool HasRegularWithIterators = false;
4813 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4814 llvm::Value *NumOfRegularWithIterators =
4815 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4816 // Calculate number of depobj dependecies and regular deps with the iterators.
4817 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4818 if (D.DepKind == OMPC_DEPEND_depobj) {
4819 SmallVector<llvm::Value *, 4> Sizes =
4820 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4821 for (llvm::Value *Size : Sizes) {
4822 NumOfDepobjElements =
4823 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4824 }
4825 HasDepobjDeps = true;
4826 continue;
4827 }
4828 // Include number of iterations, if any.
4829 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {
4830 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4831 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4832 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4833 NumOfRegularWithIterators =
4834 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz);
4835 }
4836 HasRegularWithIterators = true;
4837 continue;
4838 }
4839 }
4840
4841 QualType KmpDependInfoArrayTy;
4842 if (HasDepobjDeps || HasRegularWithIterators) {
4843 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,
4844 /*isSigned=*/false);
4845 if (HasDepobjDeps) {
4846 NumOfElements =
4847 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4848 }
4849 if (HasRegularWithIterators) {
4850 NumOfElements =
4851 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4852 }
4853 OpaqueValueExpr OVE(Loc,
4854 C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4855 VK_RValue);
4856 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE,
4857 RValue::get(NumOfElements));
4858 KmpDependInfoArrayTy =
4859 C.getVariableArrayType(KmpDependInfoTy, &OVE, ArrayType::Normal,
4860 /*IndexTypeQuals=*/0, SourceRange(Loc, Loc));
4861 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4862 // Properly emit variable-sized array.
4863 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,
4864 ImplicitParamDecl::Other);
4865 CGF.EmitVarDecl(*PD);
4866 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4867 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4868 /*isSigned=*/false);
4869 } else {
4870 KmpDependInfoArrayTy = C.getConstantArrayType(
4871 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,
4872 ArrayType::Normal, /*IndexTypeQuals=*/0);
4873 DependenciesArray =
4874 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
4875 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4876 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,
4877 /*isSigned=*/false);
4878 }
4879 unsigned Pos = 0;
4880 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4881 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4882 Dependencies[I].IteratorExpr)
4883 continue;
4884 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I],
4885 DependenciesArray);
4886 }
4887 // Copy regular dependecies with iterators.
4888 LValue PosLVal = CGF.MakeAddrLValue(
4889 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());
4890 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4891 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4892 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4893 !Dependencies[I].IteratorExpr)
4894 continue;
4895 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I],
4896 DependenciesArray);
4897 }
4898 // Copy final depobj arrays without iterators.
4899 if (HasDepobjDeps) {
4900 for (unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4901 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4902 continue;
4903 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4904 DependenciesArray);
4905 }
4906 }
4907 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4908 DependenciesArray, CGF.VoidPtrTy);
4909 return std::make_pair(NumOfElements, DependenciesArray);
4910}
4911
4912Address CGOpenMPRuntime::emitDepobjDependClause(
4913 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4914 SourceLocation Loc) {
4915 if (Dependencies.DepExprs.empty())
4916 return Address::invalid();
4917 // Process list of dependencies.
4918 ASTContext &C = CGM.getContext();
4919 Address DependenciesArray = Address::invalid();
4920 unsigned NumDependencies = Dependencies.DepExprs.size();
4921 QualType FlagsTy;
4922 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4923 RecordDecl *KmpDependInfoRD =
4924 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
4925
4926 llvm::Value *Size;
4927 // Define type kmp_depend_info[<Dependencies.size()>];
4928 // For depobj reserve one extra element to store the number of elements.
4929 // It is required to handle depobj(x) update(in) construct.
4930 // kmp_depend_info[<Dependencies.size()>] deps;
4931 llvm::Value *NumDepsVal;
4932 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4933 if (const auto *IE =
4934 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {
4935 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4936 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4937 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4938 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4939 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4940 }
4941 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4942 NumDepsVal);
4943 CharUnits SizeInBytes =
4944 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4945 llvm::Value *RecSize = CGM.getSize(SizeInBytes);
4946 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4947 NumDepsVal =
4948 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4949 } else {
4950 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4951 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4952 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
4953 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4954 Size = CGM.getSize(Sz.alignTo(Align));
4955 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4956 }
4957 // Need to allocate on the dynamic memory.
4958 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4959 // Use default allocator.
4960 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4961 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4962
4963 llvm::Value *Addr =
4964 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4965 CGM.getModule(), OMPRTL___kmpc_alloc),
4966 Args, ".dep.arr.addr");
4967 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4968 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo());
4969 DependenciesArray = Address(Addr, Align);
4970 // Write number of elements in the first element of array for depobj.
4971 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4972 // deps[i].base_addr = NumDependencies;
4973 LValue BaseAddrLVal = CGF.EmitLValueForField(
4974 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
4975 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4976 llvm::PointerUnion<unsigned *, LValue *> Pos;
4977 unsigned Idx = 1;
4978 LValue PosLVal;
4979 if (Dependencies.IteratorExpr) {
4980 PosLVal = CGF.MakeAddrLValue(
4981 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),
4982 C.getSizeType());
4983 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4984 /*IsInit=*/true);
4985 Pos = &PosLVal;
4986 } else {
4987 Pos = &Idx;
4988 }
4989 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4990 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4991 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy);
4992 return DependenciesArray;
4993}
4994
4995void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4996 SourceLocation Loc) {
4997 ASTContext &C = CGM.getContext();
4998 QualType FlagsTy;
4999 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5000 LValue Base = CGF.EmitLoadOfPointerLValue(
5001 DepobjLVal.getAddress(CGF),
5002 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5003 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
5004 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5005 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
5006 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
5007 Addr.getPointer(),
5008 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
5009 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
5010 CGF.VoidPtrTy);
5011 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5012 // Use default allocator.
5013 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5014 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
5015
5016 // _kmpc_free(gtid, addr, nullptr);
5017 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5018 CGM.getModule(), OMPRTL___kmpc_free),
5019 Args);
5020}
5021
5022void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
5023 OpenMPDependClauseKind NewDepKind,
5024 SourceLocation Loc) {
5025 ASTContext &C = CGM.getContext();
5026 QualType FlagsTy;
5027 getDependTypes(C, KmpDependInfoTy, FlagsTy);
5028 RecordDecl *KmpDependInfoRD =
5029 cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5030 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5031 llvm::Value *NumDeps;
5032 LValue Base;
5033 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
5034
5035 Address Begin = Base.getAddress(CGF);
5036 // Cast from pointer to array type to pointer to single element.
5037 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getPointer(), NumDeps);
5038 // The basic structure here is a while-do loop.
5039 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
5040 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
5041 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5042 CGF.EmitBlock(BodyBB);
5043 llvm::PHINode *ElementPHI =
5044 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
5045 ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
5046 Begin = Address(ElementPHI, Begin.getAlignment());
5047 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
5048 Base.getTBAAInfo());
5049 // deps[i].flags = NewDepKind;
5050 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
5051 LValue FlagsLVal = CGF.EmitLValueForField(
5052 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5053 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5054 FlagsLVal);
5055
5056 // Shift the address forward by one element.
5057 Address ElementNext =
5058 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
5059 ElementPHI->addIncoming(ElementNext.getPointer(),
5060 CGF.Builder.GetInsertBlock());
5061 llvm::Value *IsEmpty =
5062 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
5063 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5064 // Done.
5065 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5066}
5067
5068void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5069 const OMPExecutableDirective &D,
5070 llvm::Function *TaskFunction,
5071 QualType SharedsTy, Address Shareds,
5072 const Expr *IfCond,
5073 const OMPTaskDataTy &Data) {
5074 if (!CGF.HaveInsertPoint())
5075 return;
5076
5077 TaskResultTy Result =
5078 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5079 llvm::Value *NewTask = Result.NewTask;
5080 llvm::Function *TaskEntry = Result.TaskEntry;
5081 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5082 LValue TDBase = Result.TDBase;
5083 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5084 // Process list of dependences.
5085 Address DependenciesArray = Address::invalid();
5086 llvm::Value *NumOfElements;
5087 std::tie(NumOfElements, DependenciesArray) =
5088 emitDependClause(CGF, Data.Dependences, Loc);
5089
5090 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5091 // libcall.
5092 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5093 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5094 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5095 // list is not empty
5096 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5097 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5098 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5099 llvm::Value *DepTaskArgs[7];
5100 if (!Data.Dependences.empty()) {
5101 DepTaskArgs[0] = UpLoc;
5102 DepTaskArgs[1] = ThreadID;
5103 DepTaskArgs[2] = NewTask;
5104 DepTaskArgs[3] = NumOfElements;
5105 DepTaskArgs[4] = DependenciesArray.getPointer();
5106 DepTaskArgs[5] = CGF.Builder.getInt32(0);
5107 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5108 }
5109 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
5110 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5111 if (!Data.Tied) {
5112 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5113 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5114 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5115 }
5116 if (!Data.Dependences.empty()) {
5117 CGF.EmitRuntimeCall(
5118 OMPBuilder.getOrCreateRuntimeFunction(
5119 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
5120 DepTaskArgs);
5121 } else {
5122 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5123 CGM.getModule(), OMPRTL___kmpc_omp_task),
5124 TaskArgs);
5125 }
5126 // Check if parent region is untied and build return for untied task;
5127 if (auto *Region =
5128 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5129 Region->emitUntiedSwitch(CGF);
5130 };
5131
5132 llvm::Value *DepWaitTaskArgs[6];
5133 if (!Data.Dependences.empty()) {
5134 DepWaitTaskArgs[0] = UpLoc;
5135 DepWaitTaskArgs[1] = ThreadID;
5136 DepWaitTaskArgs[2] = NumOfElements;
5137 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5138 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5139 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5140 }
5141 auto &M = CGM.getModule();
5142 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
5143 TaskEntry, &Data, &DepWaitTaskArgs,
5144 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5145 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5146 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5147 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5148 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5149 // is specified.
5150 if (!Data.Dependences.empty())
5151 CGF.EmitRuntimeCall(
5152 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_wait_deps),
5153 DepWaitTaskArgs);
5154 // Call proxy_task_entry(gtid, new_task);
5155 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5156 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5157 Action.Enter(CGF);
5158 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5159 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5160 OutlinedFnArgs);
5161 };
5162
5163 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5164 // kmp_task_t *new_task);
5165 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5166 // kmp_task_t *new_task);
5167 RegionCodeGenTy RCG(CodeGen);
5168 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
5169 M, OMPRTL___kmpc_omp_task_begin_if0),
5170 TaskArgs,
5171 OMPBuilder.getOrCreateRuntimeFunction(
5172 M, OMPRTL___kmpc_omp_task_complete_if0),
5173 TaskArgs);
5174 RCG.setAction(Action);
5175 RCG(CGF);
5176 };
5177
5178 if (IfCond) {
5179 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5180 } else {
5181 RegionCodeGenTy ThenRCG(ThenCodeGen);
5182 ThenRCG(CGF);
5183 }
5184}
5185
5186void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5187 const OMPLoopDirective &D,
5188 llvm::Function *TaskFunction,
5189 QualType SharedsTy, Address Shareds,
5190 const Expr *IfCond,
5191 const OMPTaskDataTy &Data) {
5192 if (!CGF.HaveInsertPoint())
5193 return;
5194 TaskResultTy Result =
5195 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5196 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5197 // libcall.
5198 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5199 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5200 // sched, kmp_uint64 grainsize, void *task_dup);
5201 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5202 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5203 llvm::Value *IfVal;
5204 if (IfCond) {
5205 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5206 /*isSigned=*/true);
5207 } else {
5208 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5209 }
5210
5211 LValue LBLVal = CGF.EmitLValueForField(
5212 Result.TDBase,
5213 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5214 const auto *LBVar =
5215 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5216 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5217 LBLVal.getQuals(),
5218 /*IsInitializer=*/true);
5219 LValue UBLVal = CGF.EmitLValueForField(
5220 Result.TDBase,
5221 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5222 const auto *UBVar =
5223 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5224 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5225 UBLVal.getQuals(),
5226 /*IsInitializer=*/true);
5227 LValue StLVal = CGF.EmitLValueForField(
5228 Result.TDBase,
5229 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5230 const auto *StVar =
5231 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5232 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5233 StLVal.getQuals(),
5234 /*IsInitializer=*/true);
5235 // Store reductions address.
5236 LValue RedLVal = CGF.EmitLValueForField(
5237 Result.TDBase,
5238 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5239 if (Data.Reductions) {
5240 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5241 } else {
5242 CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5243 CGF.getContext().VoidPtrTy);
5244 }
5245 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5246 llvm::Value *TaskArgs[] = {
5247 UpLoc,
5248 ThreadID,
5249 Result.NewTask,
5250 IfVal,
5251 LBLVal.getPointer(CGF),
5252 UBLVal.getPointer(CGF),
5253 CGF.EmitLoadOfScalar(StLVal, Loc),
5254 llvm::ConstantInt::getSigned(
5255 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5256 llvm::ConstantInt::getSigned(
5257 CGF.IntTy, Data.Schedule.getPointer()
5258 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5259 : NoSchedule),
5260 Data.Schedule.getPointer()
5261 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5262 /*isSigned=*/false)
5263 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5264 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5265 Result.TaskDupFn, CGF.VoidPtrTy)
5266 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5267 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5268 CGM.getModule(), OMPRTL___kmpc_taskloop),
5269 TaskArgs);
5270}
5271
5272/// Emit reduction operation for each element of array (required for
5273/// array sections) LHS op = RHS.
5274/// \param Type Type of array.
5275/// \param LHSVar Variable on the left side of the reduction operation
5276/// (references element of array in original variable).
5277/// \param RHSVar Variable on the right side of the reduction operation
5278/// (references element of array in original variable).
5279/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5280/// RHSVar.
5281static void EmitOMPAggregateReduction(
5282 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5283 const VarDecl *RHSVar,
5284 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5285 const Expr *, const Expr *)> &RedOpGen,
5286 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5287 const Expr *UpExpr = nullptr) {
5288 // Perform element-by-element initialization.
5289 QualType ElementTy;
5290 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5291 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5292
5293 // Drill down to the base element type on both arrays.
5294 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5295 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5296
5297 llvm::Value *RHSBegin = RHSAddr.getPointer();
5298 llvm::Value *LHSBegin = LHSAddr.getPointer();
5299 // Cast from pointer to array type to pointer to single element.
5300 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5301 // The basic structure here is a while-do loop.
5302 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5303 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5304 llvm::Value *IsEmpty =
5305 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5306 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5307
5308 // Enter the loop body, making that address the current address.
5309 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5310 CGF.EmitBlock(BodyBB);
5311
5312 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5313
5314 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5315 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5316 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5317 Address RHSElementCurrent =
5318 Address(RHSElementPHI,
5319 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5320
5321 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5322 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5323 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5324 Address LHSElementCurrent =
5325 Address(LHSElementPHI,
5326 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5327
5328 // Emit copy.
5329 CodeGenFunction::OMPPrivateScope Scope(CGF);
5330 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5331 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5332 Scope.Privatize();
5333 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5334 Scope.ForceCleanup();
5335
5336 // Shift the address forward by one element.
5337 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5338 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5339 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5340 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5341 // Check whether we've reached the end.
5342 llvm::Value *Done =
5343 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5344 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5345 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5346 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5347
5348 // Done.
5349 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5350}
5351
5352/// Emit reduction combiner. If the combiner is a simple expression emit it as
5353/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5354/// UDR combiner function.
5355static void emitReductionCombiner(CodeGenFunction &CGF,
5356 const Expr *ReductionOp) {
5357 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5358 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5359 if (const auto *DRE =
5360 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5361 if (const auto *DRD =
5362 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5363 std::pair<llvm::Function *, llvm::Function *> Reduction =
5364 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5365 RValue Func = RValue::get(Reduction.first);
5366 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5367 CGF.EmitIgnoredExpr(ReductionOp);
5368 return;
5369 }
5370 CGF.EmitIgnoredExpr(ReductionOp);
5371}
5372
5373llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5374 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5375 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5376 ArrayRef<const Expr *> ReductionOps) {
5377 ASTContext &C = CGM.getContext();
5378
5379 // void reduction_func(void *LHSArg, void *RHSArg);
5380 FunctionArgList Args;
5381 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5382 ImplicitParamDecl::Other);
5383 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5384 ImplicitParamDecl::Other);
5385 Args.push_back(&LHSArg);
5386 Args.push_back(&RHSArg);
5387 const auto &CGFI =
5388 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5389 std::string Name = getName({"omp", "reduction", "reduction_func"});
5390 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5391 llvm::GlobalValue::InternalLinkage, Name,
5392 &CGM.getModule());
5393 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5394 Fn->setDoesNotRecurse();
5395 CodeGenFunction CGF(CGM);
5396 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5397
5398 // Dst = (void*[n])(LHSArg);
5399 // Src = (void*[n])(RHSArg);
5400 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5401 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5402 ArgsType), CGF.getPointerAlign());
5403 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5404 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5405 ArgsType), CGF.getPointerAlign());
5406
5407 // ...
5408 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5409 // ...
5410 CodeGenFunction::OMPPrivateScope Scope(CGF);
5411 auto IPriv = Privates.begin();
5412 unsigned Idx = 0;
5413 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5414 const auto *RHSVar =
5415 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5416 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5417 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5418 });
5419 const auto *LHSVar =
5420 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5421 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5422 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5423 });
5424 QualType PrivTy = (*IPriv)->getType();
5425 if (PrivTy->isVariablyModifiedType()) {
5426 // Get array size and emit VLA type.
5427 ++Idx;
5428 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5429 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5430 const VariableArrayType *VLA =
5431 CGF.getContext().getAsVariableArrayType(PrivTy);
5432 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5433 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5434 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5435 CGF.EmitVariablyModifiedType(PrivTy);
5436 }
5437 }
5438 Scope.Privatize();
5439 IPriv = Privates.begin();
5440 auto ILHS = LHSExprs.begin();
5441 auto IRHS = RHSExprs.begin();
5442 for (const Expr *E : ReductionOps) {
5443 if ((*IPriv)->getType()->isArrayType()) {
5444 // Emit reduction for array section.
5445 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5446 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5447 EmitOMPAggregateReduction(
5448 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5449 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5450 emitReductionCombiner(CGF, E);
5451 });
5452 } else {
5453 // Emit reduction for array subscript or single variable.
5454 emitReductionCombiner(CGF, E);
5455 }
5456 ++IPriv;
5457 ++ILHS;
5458 ++IRHS;
5459 }
5460 Scope.ForceCleanup();
5461 CGF.FinishFunction();
5462 return Fn;
5463}
5464
5465void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5466 const Expr *ReductionOp,
5467 const Expr *PrivateRef,
5468 const DeclRefExpr *LHS,
5469 const DeclRefExpr *RHS) {
5470 if (PrivateRef->getType()->isArrayType()) {
5471 // Emit reduction for array section.
5472 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5473 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5474 EmitOMPAggregateReduction(
5475 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5476 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5477 emitReductionCombiner(CGF, ReductionOp);
5478 });
5479 } else {
5480 // Emit reduction for array subscript or single variable.
5481 emitReductionCombiner(CGF, ReductionOp);
5482 }
5483}
5484
5485void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5486 ArrayRef<const Expr *> Privates,
5487 ArrayRef<const Expr *> LHSExprs,
5488 ArrayRef<const Expr *> RHSExprs,
5489 ArrayRef<const Expr *> ReductionOps,
5490 ReductionOptionsTy Options) {
5491 if (!CGF.HaveInsertPoint())
5492 return;
5493
5494 bool WithNowait = Options.WithNowait;
5495 bool SimpleReduction = Options.SimpleReduction;
5496
5497 // Next code should be emitted for reduction:
5498 //
5499 // static kmp_critical_name lock = { 0 };
5500 //
5501 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5502 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5503 // ...
5504 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5505 // *(Type<n>-1*)rhs[<n>-1]);
5506 // }
5507 //
5508 // ...
5509 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5510 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5511 // RedList, reduce_func, &<lock>)) {
5512 // case 1:
5513 // ...
5514 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5515 // ...
5516 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5517 // break;
5518 // case 2:
5519 // ...
5520 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5521 // ...
5522 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5523 // break;
5524 // default:;
5525 // }
5526 //
5527 // if SimpleReduction is true, only the next code is generated:
5528 // ...
5529 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5530 // ...
5531
5532 ASTContext &C = CGM.getContext();
5533
5534 if (SimpleReduction) {
5535 CodeGenFunction::RunCleanupsScope Scope(CGF);
5536 auto IPriv = Privates.begin();
5537 auto ILHS = LHSExprs.begin();
5538 auto IRHS = RHSExprs.begin();
5539 for (const Expr *E : ReductionOps) {
5540 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5541 cast<DeclRefExpr>(*IRHS));
5542 ++IPriv;
5543 ++ILHS;
5544 ++IRHS;
5545 }
5546 return;
5547 }
5548
5549 // 1. Build a list of reduction variables.
5550 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5551 auto Size = RHSExprs.size();
5552 for (const Expr *E : Privates) {
5553 if (E->getType()->isVariablyModifiedType())
5554 // Reserve place for array size.
5555 ++Size;
5556 }
5557 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5558 QualType ReductionArrayTy =
5559 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5560 /*IndexTypeQuals=*/0);
5561 Address ReductionList =
5562 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5563 auto IPriv = Privates.begin();
5564 unsigned Idx = 0;
5565 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5566 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5567 CGF.Builder.CreateStore(
5568 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5569 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5570 Elem);
5571 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5572 // Store array size.
5573 ++Idx;
5574 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5575 llvm::Value *Size = CGF.Builder.CreateIntCast(
5576 CGF.getVLASize(
5577 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5578 .NumElts,
5579 CGF.SizeTy, /*isSigned=*/false);
5580 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5581 Elem);
5582 }
5583 }
5584
5585 // 2. Emit reduce_func().
5586 llvm::Function *ReductionFn = emitReductionFunction(
5587 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5588 LHSExprs, RHSExprs, ReductionOps);
5589
5590 // 3. Create static kmp_critical_name lock = { 0 };
5591 std::string Name = getName({"reduction"});
5592 llvm::Value *Lock = getCriticalRegionLock(Name);
5593
5594 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5595 // RedList, reduce_func, &<lock>);
5596 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5597 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5598 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5599 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5600 ReductionList.getPointer(), CGF.VoidPtrTy);
5601 llvm::Value *Args[] = {
5602 IdentTLoc, // ident_t *<loc>
5603 ThreadId, // i32 <gtid>
5604 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5605 ReductionArrayTySize, // size_type sizeof(RedList)
5606 RL, // void *RedList
5607 ReductionFn, // void (*) (void *, void *) <reduce_func>
5608 Lock // kmp_critical_name *&<lock>
5609 };
5610 llvm::Value *Res = CGF.EmitRuntimeCall(
5611 OMPBuilder.getOrCreateRuntimeFunction(
5612 CGM.getModule(),
5613 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5614 Args);
5615
5616 // 5. Build switch(res)
5617 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5618 llvm::SwitchInst *SwInst =
5619 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5620
5621 // 6. Build case 1:
5622 // ...
5623 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5624 // ...
5625 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5626 // break;
5627 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5628 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5629 CGF.EmitBlock(Case1BB);
5630
5631 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5632 llvm::Value *EndArgs[] = {
5633 IdentTLoc, // ident_t *<loc>
5634 ThreadId, // i32 <gtid>
5635 Lock // kmp_critical_name *&<lock>
5636 };
5637 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5638 CodeGenFunction &CGF, PrePostActionTy &Action) {
5639 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5640 auto IPriv = Privates.begin();
5641 auto ILHS = LHSExprs.begin();
5642 auto IRHS = RHSExprs.begin();
5643 for (const Expr *E : ReductionOps) {
5644 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5645 cast<DeclRefExpr>(*IRHS));
5646 ++IPriv;
5647 ++ILHS;
5648 ++IRHS;
5649 }
5650 };
5651 RegionCodeGenTy RCG(CodeGen);
5652 CommonActionTy Action(
5653 nullptr, llvm::None,
5654 OMPBuilder.getOrCreateRuntimeFunction(
5655 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5656 : OMPRTL___kmpc_end_reduce),
5657 EndArgs);
5658 RCG.setAction(Action);
5659 RCG(CGF);
5660
5661 CGF.EmitBranch(DefaultBB);
5662
5663 // 7. Build case 2:
5664 // ...
5665 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5666 // ...
5667 // break;
5668 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5669 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5670 CGF.EmitBlock(Case2BB);
5671
5672 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5673 CodeGenFunction &CGF, PrePostActionTy &Action) {
5674 auto ILHS = LHSExprs.begin();
5675 auto IRHS = RHSExprs.begin();
5676 auto IPriv = Privates.begin();
5677 for (const Expr *E : ReductionOps) {
5678 const Expr *XExpr = nullptr;
5679 const Expr *EExpr = nullptr;
5680 const Expr *UpExpr = nullptr;
5681 BinaryOperatorKind BO = BO_Comma;
5682 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5683 if (BO->getOpcode() == BO_Assign) {
5684 XExpr = BO->getLHS();
5685 UpExpr = BO->getRHS();
5686 }
5687 }
5688 // Try to emit update expression as a simple atomic.
5689 const Expr *RHSExpr = UpExpr;
5690 if (RHSExpr) {
5691 // Analyze RHS part of the whole expression.
5692 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5693 RHSExpr->IgnoreParenImpCasts())) {
5694 // If this is a conditional operator, analyze its condition for
5695 // min/max reduction operator.
5696 RHSExpr = ACO->getCond();
5697 }
5698 if (const auto *BORHS =
5699 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5700 EExpr = BORHS->getRHS();
5701 BO = BORHS->getOpcode();
5702 }
5703 }
5704 if (XExpr) {
5705 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5706 auto &&AtomicRedGen = [BO, VD,
5707 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5708 const Expr *EExpr, const Expr *UpExpr) {
5709 LValue X = CGF.EmitLValue(XExpr);
5710 RValue E;
5711 if (EExpr)
5712 E = CGF.EmitAnyExpr(EExpr);
5713 CGF.EmitOMPAtomicSimpleUpdateExpr(
5714 X, E, BO, /*IsXLHSInRHSPart=*/true,
5715 llvm::AtomicOrdering::Monotonic, Loc,
5716 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5717 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5718 PrivateScope.addPrivate(
5719 VD, [&CGF, VD, XRValue, Loc]() {
5720 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5721 CGF.emitOMPSimpleStore(
5722 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5723 VD->getType().getNonReferenceType(), Loc);
5724 return LHSTemp;
5725 });
5726 (void)PrivateScope.Privatize();
5727 return CGF.EmitAnyExpr(UpExpr);
5728 });
5729 };
5730 if ((*IPriv)->getType()->isArrayType()) {
5731 // Emit atomic reduction for array section.
5732 const auto *RHSVar =
5733 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5734 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5735 AtomicRedGen, XExpr, EExpr, UpExpr);
5736 } else {
5737 // Emit atomic reduction for array subscript or single variable.
5738 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5739 }
5740 } else {
5741 // Emit as a critical region.
5742 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5743 const Expr *, const Expr *) {
5744 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5745 std::string Name = RT.getName({"atomic_reduction"});
5746 RT.emitCriticalRegion(
5747 CGF, Name,
5748 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5749 Action.Enter(CGF);
5750 emitReductionCombiner(CGF, E);
5751 },
5752 Loc);
5753 };
5754 if ((*IPriv)->getType()->isArrayType()) {
5755 const auto *LHSVar =
5756 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5757 const auto *RHSVar =
5758 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5759 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5760 CritRedGen);
5761 } else {
5762 CritRedGen(CGF, nullptr, nullptr, nullptr);
5763 }
5764 }
5765 ++ILHS;
5766 ++IRHS;
5767 ++IPriv;
5768 }
5769 };
5770 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5771 if (!WithNowait) {
5772 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5773 llvm::Value *EndArgs[] = {
5774 IdentTLoc, // ident_t *<loc>
5775 ThreadId, // i32 <gtid>
5776 Lock // kmp_critical_name *&<lock>
5777 };
5778 CommonActionTy Action(nullptr, llvm::None,
5779 OMPBuilder.getOrCreateRuntimeFunction(
5780 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5781 EndArgs);
5782 AtomicRCG.setAction(Action);
5783 AtomicRCG(CGF);
5784 } else {
5785 AtomicRCG(CGF);
5786 }
5787
5788 CGF.EmitBranch(DefaultBB);
5789 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5790}
5791
5792/// Generates unique name for artificial threadprivate variables.
5793/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5794static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5795 const Expr *Ref) {
5796 SmallString<256> Buffer;
5797 llvm::raw_svector_ostream Out(Buffer);
5798 const clang::DeclRefExpr *DE;
5799 const VarDecl *D = ::getBaseDecl(Ref, DE);
5800 if (!D)
5801 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5802 D = D->getCanonicalDecl();
5803 std::string Name = CGM.getOpenMPRuntime().getName(
5804 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5805 Out << Prefix << Name << "_"
5806 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5807 return std::string(Out.str());
5808}
5809
5810/// Emits reduction initializer function:
5811/// \code
5812/// void @.red_init(void* %arg, void* %orig) {
5813/// %0 = bitcast void* %arg to <type>*
5814/// store <type> <init>, <type>* %0
5815/// ret void
5816/// }
5817/// \endcode
5818static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5819 SourceLocation Loc,
5820 ReductionCodeGen &RCG, unsigned N) {
5821 ASTContext &C = CGM.getContext();
5822