Bug Summary

File:clang/lib/CodeGen/CGStmtOpenMP.cpp
Warning:line 5237, column 16
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGStmtOpenMP.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/lib/CodeGen -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-07-102640-14763-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/Stmt.h"
19#include "clang/AST/StmtOpenMP.h"
20#include "clang/AST/DeclOpenMP.h"
21#include "clang/Basic/PrettyStackTrace.h"
22using namespace clang;
23using namespace CodeGen;
24
25namespace {
26/// Lexical scope for OpenMP executable constructs, that handles correct codegen
27/// for captured expressions.
28class OMPLexicalScope : public CodeGenFunction::LexicalScope {
29 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
30 for (const auto *C : S.clauses()) {
31 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
32 if (const auto *PreInit =
33 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34 for (const auto *I : PreInit->decls()) {
35 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
36 CGF.EmitVarDecl(cast<VarDecl>(*I));
37 } else {
38 CodeGenFunction::AutoVarEmission Emission =
39 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40 CGF.EmitAutoVarCleanups(Emission);
41 }
42 }
43 }
44 }
45 }
46 }
47 CodeGenFunction::OMPPrivateScope InlinedShareds;
48
49 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50 return CGF.LambdaCaptureFields.lookup(VD) ||
51 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
53 }
54
55public:
56 OMPLexicalScope(
57 CodeGenFunction &CGF, const OMPExecutableDirective &S,
58 const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
59 const bool EmitPreInitStmt = true)
60 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
61 InlinedShareds(CGF) {
62 if (EmitPreInitStmt)
63 emitPreInitStmt(CGF, S);
64 if (!CapturedRegion.hasValue())
65 return;
66 assert(S.hasAssociatedStmt() &&((S.hasAssociatedStmt() && "Expected associated statement for inlined directive."
) ? static_cast<void> (0) : __assert_fail ("S.hasAssociatedStmt() && \"Expected associated statement for inlined directive.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 67, __PRETTY_FUNCTION__))
67 "Expected associated statement for inlined directive.")((S.hasAssociatedStmt() && "Expected associated statement for inlined directive."
) ? static_cast<void> (0) : __assert_fail ("S.hasAssociatedStmt() && \"Expected associated statement for inlined directive.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 67, __PRETTY_FUNCTION__))
;
68 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
69 for (const auto &C : CS->captures()) {
70 if (C.capturesVariable() || C.capturesVariableByCopy()) {
71 auto *VD = C.getCapturedVar();
72 assert(VD == VD->getCanonicalDecl() &&((VD == VD->getCanonicalDecl() && "Canonical decl must be captured."
) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 73, __PRETTY_FUNCTION__))
73 "Canonical decl must be captured.")((VD == VD->getCanonicalDecl() && "Canonical decl must be captured."
) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 73, __PRETTY_FUNCTION__))
;
74 DeclRefExpr DRE(
75 CGF.getContext(), const_cast<VarDecl *>(VD),
76 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
77 InlinedShareds.isGlobalVarCaptured(VD)),
78 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
79 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
80 return CGF.EmitLValue(&DRE).getAddress();
81 });
82 }
83 }
84 (void)InlinedShareds.Privatize();
85 }
86};
87
88/// Lexical scope for OpenMP parallel construct, that handles correct codegen
89/// for captured expressions.
90class OMPParallelScope final : public OMPLexicalScope {
91 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92 OpenMPDirectiveKind Kind = S.getDirectiveKind();
93 return !(isOpenMPTargetExecutionDirective(Kind) ||
94 isOpenMPLoopBoundSharingDirective(Kind)) &&
95 isOpenMPParallelDirective(Kind);
96 }
97
98public:
99 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
100 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
101 EmitPreInitStmt(S)) {}
102};
103
104/// Lexical scope for OpenMP teams construct, that handles correct codegen
105/// for captured expressions.
106class OMPTeamsScope final : public OMPLexicalScope {
107 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108 OpenMPDirectiveKind Kind = S.getDirectiveKind();
109 return !isOpenMPTargetExecutionDirective(Kind) &&
110 isOpenMPTeamsDirective(Kind);
111 }
112
113public:
114 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
115 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
116 EmitPreInitStmt(S)) {}
117};
118
119/// Private scope for OpenMP loop-based directives, that supports capturing
120/// of used expression from loop statement.
121class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
122 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
123 CodeGenFunction::OMPMapVars PreCondVars;
124 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
125 for (const auto *E : S.counters()) {
126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127 EmittedAsPrivate.insert(VD->getCanonicalDecl());
128 (void)PreCondVars.setVarAddr(
129 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
130 }
131 // Mark private vars as undefs.
132 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
133 for (const Expr *IRef : C->varlists()) {
134 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
135 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
136 (void)PreCondVars.setVarAddr(
137 CGF, OrigVD,
138 Address(llvm::UndefValue::get(
139 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(
140 OrigVD->getType().getNonReferenceType()))),
141 CGF.getContext().getDeclAlign(OrigVD)));
142 }
143 }
144 }
145 (void)PreCondVars.apply(CGF);
146 // Emit init, __range and __end variables for C++ range loops.
147 const Stmt *Body =
148 S.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
149 for (unsigned Cnt = 0; Cnt < S.getCollapsedNumber(); ++Cnt) {
150 Body = OMPLoopDirective::tryToFindNextInnerLoop(
151 Body, /*TryImperfectlyNestedLoops=*/true);
152 if (auto *For = dyn_cast<ForStmt>(Body)) {
153 Body = For->getBody();
154 } else {
155 assert(isa<CXXForRangeStmt>(Body) &&((isa<CXXForRangeStmt>(Body) && "Expected canonical for loop or range-based for loop."
) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(Body) && \"Expected canonical for loop or range-based for loop.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 156, __PRETTY_FUNCTION__))
156 "Expected canonical for loop or range-based for loop.")((isa<CXXForRangeStmt>(Body) && "Expected canonical for loop or range-based for loop."
) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(Body) && \"Expected canonical for loop or range-based for loop.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 156, __PRETTY_FUNCTION__))
;
157 auto *CXXFor = cast<CXXForRangeStmt>(Body);
158 if (const Stmt *Init = CXXFor->getInit())
159 CGF.EmitStmt(Init);
160 CGF.EmitStmt(CXXFor->getRangeStmt());
161 CGF.EmitStmt(CXXFor->getEndStmt());
162 Body = CXXFor->getBody();
163 }
164 }
165 if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
166 for (const auto *I : PreInits->decls())
167 CGF.EmitVarDecl(cast<VarDecl>(*I));
168 }
169 PreCondVars.restore(CGF);
170 }
171
172public:
173 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
174 : CodeGenFunction::RunCleanupsScope(CGF) {
175 emitPreInitStmt(CGF, S);
176 }
177};
178
179class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
180 CodeGenFunction::OMPPrivateScope InlinedShareds;
181
182 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
183 return CGF.LambdaCaptureFields.lookup(VD) ||
184 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
185 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
186 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
187 }
188
189public:
190 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
191 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
192 InlinedShareds(CGF) {
193 for (const auto *C : S.clauses()) {
194 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
195 if (const auto *PreInit =
196 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
197 for (const auto *I : PreInit->decls()) {
198 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
199 CGF.EmitVarDecl(cast<VarDecl>(*I));
200 } else {
201 CodeGenFunction::AutoVarEmission Emission =
202 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
203 CGF.EmitAutoVarCleanups(Emission);
204 }
205 }
206 }
207 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
208 for (const Expr *E : UDP->varlists()) {
209 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
210 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
211 CGF.EmitVarDecl(*OED);
212 }
213 }
214 }
215 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
216 CGF.EmitOMPPrivateClause(S, InlinedShareds);
217 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
218 if (const Expr *E = TG->getReductionRef())
219 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
220 }
221 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
222 while (CS) {
223 for (auto &C : CS->captures()) {
224 if (C.capturesVariable() || C.capturesVariableByCopy()) {
225 auto *VD = C.getCapturedVar();
226 assert(VD == VD->getCanonicalDecl() &&((VD == VD->getCanonicalDecl() && "Canonical decl must be captured."
) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 227, __PRETTY_FUNCTION__))
227 "Canonical decl must be captured.")((VD == VD->getCanonicalDecl() && "Canonical decl must be captured."
) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 227, __PRETTY_FUNCTION__))
;
228 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
229 isCapturedVar(CGF, VD) ||
230 (CGF.CapturedStmtInfo &&
231 InlinedShareds.isGlobalVarCaptured(VD)),
232 VD->getType().getNonReferenceType(), VK_LValue,
233 C.getLocation());
234 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
235 return CGF.EmitLValue(&DRE).getAddress();
236 });
237 }
238 }
239 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
240 }
241 (void)InlinedShareds.Privatize();
242 }
243};
244
245} // namespace
246
247static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
248 const OMPExecutableDirective &S,
249 const RegionCodeGenTy &CodeGen);
250
251LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
252 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
253 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
254 OrigVD = OrigVD->getCanonicalDecl();
255 bool IsCaptured =
256 LambdaCaptureFields.lookup(OrigVD) ||
257 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
258 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
259 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
260 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
261 return EmitLValue(&DRE);
262 }
263 }
264 return EmitLValue(E);
265}
266
267llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
268 ASTContext &C = getContext();
269 llvm::Value *Size = nullptr;
270 auto SizeInChars = C.getTypeSizeInChars(Ty);
271 if (SizeInChars.isZero()) {
272 // getTypeSizeInChars() returns 0 for a VLA.
273 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
274 VlaSizePair VlaSize = getVLASize(VAT);
275 Ty = VlaSize.Type;
276 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
277 : VlaSize.NumElts;
278 }
279 SizeInChars = C.getTypeSizeInChars(Ty);
280 if (SizeInChars.isZero())
281 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
282 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
283 }
284 return CGM.getSize(SizeInChars);
285}
286
287void CodeGenFunction::GenerateOpenMPCapturedVars(
288 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
289 const RecordDecl *RD = S.getCapturedRecordDecl();
290 auto CurField = RD->field_begin();
291 auto CurCap = S.captures().begin();
292 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
293 E = S.capture_init_end();
294 I != E; ++I, ++CurField, ++CurCap) {
295 if (CurField->hasCapturedVLAType()) {
296 const VariableArrayType *VAT = CurField->getCapturedVLAType();
297 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
298 CapturedVars.push_back(Val);
299 } else if (CurCap->capturesThis()) {
300 CapturedVars.push_back(CXXThisValue);
301 } else if (CurCap->capturesVariableByCopy()) {
302 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
303
304 // If the field is not a pointer, we need to save the actual value
305 // and load it as a void pointer.
306 if (!CurField->getType()->isAnyPointerType()) {
307 ASTContext &Ctx = getContext();
308 Address DstAddr = CreateMemTemp(
309 Ctx.getUIntPtrType(),
310 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
311 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
312
313 llvm::Value *SrcAddrVal = EmitScalarConversion(
314 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
315 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
316 LValue SrcLV =
317 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
318
319 // Store the value using the source type pointer.
320 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
321
322 // Load the value using the destination type pointer.
323 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
324 }
325 CapturedVars.push_back(CV);
326 } else {
327 assert(CurCap->capturesVariable() && "Expected capture by reference.")((CurCap->capturesVariable() && "Expected capture by reference."
) ? static_cast<void> (0) : __assert_fail ("CurCap->capturesVariable() && \"Expected capture by reference.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 327, __PRETTY_FUNCTION__))
;
328 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
329 }
330 }
331}
332
333static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
334 QualType DstType, StringRef Name,
335 LValue AddrLV) {
336 ASTContext &Ctx = CGF.getContext();
337
338 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
339 AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
340 Ctx.getPointerType(DstType), Loc);
341 Address TmpAddr =
342 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
343 .getAddress();
344 return TmpAddr;
345}
346
347static QualType getCanonicalParamType(ASTContext &C, QualType T) {
348 if (T->isLValueReferenceType())
349 return C.getLValueReferenceType(
350 getCanonicalParamType(C, T.getNonReferenceType()),
351 /*SpelledAsLValue=*/false);
352 if (T->isPointerType())
353 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
354 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
355 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
356 return getCanonicalParamType(C, VLA->getElementType());
357 if (!A->isVariablyModifiedType())
358 return C.getCanonicalType(T);
359 }
360 return C.getCanonicalParamType(T);
361}
362
363namespace {
364 /// Contains required data for proper outlined function codegen.
365 struct FunctionOptions {
366 /// Captured statement for which the function is generated.
367 const CapturedStmt *S = nullptr;
368 /// true if cast to/from UIntPtr is required for variables captured by
369 /// value.
370 const bool UIntPtrCastRequired = true;
371 /// true if only casted arguments must be registered as local args or VLA
372 /// sizes.
373 const bool RegisterCastedArgsOnly = false;
374 /// Name of the generated function.
375 const StringRef FunctionName;
376 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
377 bool RegisterCastedArgsOnly,
378 StringRef FunctionName)
379 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
380 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
381 FunctionName(FunctionName) {}
382 };
383}
384
385static llvm::Function *emitOutlinedFunctionPrologue(
386 CodeGenFunction &CGF, FunctionArgList &Args,
387 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
388 &LocalAddrs,
389 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
390 &VLASizes,
391 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
392 const CapturedDecl *CD = FO.S->getCapturedDecl();
393 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
394 assert(CD->hasBody() && "missing CapturedDecl body")((CD->hasBody() && "missing CapturedDecl body") ? static_cast
<void> (0) : __assert_fail ("CD->hasBody() && \"missing CapturedDecl body\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 394, __PRETTY_FUNCTION__))
;
395
396 CXXThisValue = nullptr;
397 // Build the argument list.
398 CodeGenModule &CGM = CGF.CGM;
399 ASTContext &Ctx = CGM.getContext();
400 FunctionArgList TargetArgs;
401 Args.append(CD->param_begin(),
402 std::next(CD->param_begin(), CD->getContextParamPosition()));
403 TargetArgs.append(
404 CD->param_begin(),
405 std::next(CD->param_begin(), CD->getContextParamPosition()));
406 auto I = FO.S->captures().begin();
407 FunctionDecl *DebugFunctionDecl = nullptr;
408 if (!FO.UIntPtrCastRequired) {
409 FunctionProtoType::ExtProtoInfo EPI;
410 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
411 DebugFunctionDecl = FunctionDecl::Create(
412 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
413 SourceLocation(), DeclarationName(), FunctionTy,
414 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
415 /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
416 }
417 for (const FieldDecl *FD : RD->fields()) {
418 QualType ArgType = FD->getType();
419 IdentifierInfo *II = nullptr;
420 VarDecl *CapVar = nullptr;
421
422 // If this is a capture by copy and the type is not a pointer, the outlined
423 // function argument type should be uintptr and the value properly casted to
424 // uintptr. This is necessary given that the runtime library is only able to
425 // deal with pointers. We can pass in the same way the VLA type sizes to the
426 // outlined function.
427 if (FO.UIntPtrCastRequired &&
428 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
429 I->capturesVariableArrayType()))
430 ArgType = Ctx.getUIntPtrType();
431
432 if (I->capturesVariable() || I->capturesVariableByCopy()) {
433 CapVar = I->getCapturedVar();
434 II = CapVar->getIdentifier();
435 } else if (I->capturesThis()) {
436 II = &Ctx.Idents.get("this");
437 } else {
438 assert(I->capturesVariableArrayType())((I->capturesVariableArrayType()) ? static_cast<void>
(0) : __assert_fail ("I->capturesVariableArrayType()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 438, __PRETTY_FUNCTION__))
;
439 II = &Ctx.Idents.get("vla");
440 }
441 if (ArgType->isVariablyModifiedType())
442 ArgType = getCanonicalParamType(Ctx, ArgType);
443 VarDecl *Arg;
444 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
445 Arg = ParmVarDecl::Create(
446 Ctx, DebugFunctionDecl,
447 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
448 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
449 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
450 } else {
451 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
452 II, ArgType, ImplicitParamDecl::Other);
453 }
454 Args.emplace_back(Arg);
455 // Do not cast arguments if we emit function with non-original types.
456 TargetArgs.emplace_back(
457 FO.UIntPtrCastRequired
458 ? Arg
459 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
460 ++I;
461 }
462 Args.append(
463 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
464 CD->param_end());
465 TargetArgs.append(
466 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
467 CD->param_end());
468
469 // Create the function declaration.
470 const CGFunctionInfo &FuncInfo =
471 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
472 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
473
474 auto *F =
475 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
476 FO.FunctionName, &CGM.getModule());
477 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
478 if (CD->isNothrow())
479 F->setDoesNotThrow();
480 F->setDoesNotRecurse();
481
482 // Generate the function.
483 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
484 FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
485 unsigned Cnt = CD->getContextParamPosition();
486 I = FO.S->captures().begin();
487 for (const FieldDecl *FD : RD->fields()) {
488 // Do not map arguments if we emit function with non-original types.
489 Address LocalAddr(Address::invalid());
490 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
491 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
492 TargetArgs[Cnt]);
493 } else {
494 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
495 }
496 // If we are capturing a pointer by copy we don't need to do anything, just
497 // use the value that we get from the arguments.
498 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
499 const VarDecl *CurVD = I->getCapturedVar();
500 if (!FO.RegisterCastedArgsOnly)
501 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
502 ++Cnt;
503 ++I;
504 continue;
505 }
506
507 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
508 AlignmentSource::Decl);
509 if (FD->hasCapturedVLAType()) {
510 if (FO.UIntPtrCastRequired) {
511 ArgLVal = CGF.MakeAddrLValue(
512 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
513 Args[Cnt]->getName(), ArgLVal),
514 FD->getType(), AlignmentSource::Decl);
515 }
516 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
517 const VariableArrayType *VAT = FD->getCapturedVLAType();
518 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
519 } else if (I->capturesVariable()) {
520 const VarDecl *Var = I->getCapturedVar();
521 QualType VarTy = Var->getType();
522 Address ArgAddr = ArgLVal.getAddress();
523 if (ArgLVal.getType()->isLValueReferenceType()) {
524 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
525 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
526 assert(ArgLVal.getType()->isPointerType())((ArgLVal.getType()->isPointerType()) ? static_cast<void
> (0) : __assert_fail ("ArgLVal.getType()->isPointerType()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 526, __PRETTY_FUNCTION__))
;
527 ArgAddr = CGF.EmitLoadOfPointer(
528 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
529 }
530 if (!FO.RegisterCastedArgsOnly) {
531 LocalAddrs.insert(
532 {Args[Cnt],
533 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
534 }
535 } else if (I->capturesVariableByCopy()) {
536 assert(!FD->getType()->isAnyPointerType() &&((!FD->getType()->isAnyPointerType() && "Not expecting a captured pointer."
) ? static_cast<void> (0) : __assert_fail ("!FD->getType()->isAnyPointerType() && \"Not expecting a captured pointer.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 537, __PRETTY_FUNCTION__))
537 "Not expecting a captured pointer.")((!FD->getType()->isAnyPointerType() && "Not expecting a captured pointer."
) ? static_cast<void> (0) : __assert_fail ("!FD->getType()->isAnyPointerType() && \"Not expecting a captured pointer.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 537, __PRETTY_FUNCTION__))
;
538 const VarDecl *Var = I->getCapturedVar();
539 LocalAddrs.insert({Args[Cnt],
540 {Var, FO.UIntPtrCastRequired
541 ? castValueFromUintptr(
542 CGF, I->getLocation(), FD->getType(),
543 Args[Cnt]->getName(), ArgLVal)
544 : ArgLVal.getAddress()}});
545 } else {
546 // If 'this' is captured, load it into CXXThisValue.
547 assert(I->capturesThis())((I->capturesThis()) ? static_cast<void> (0) : __assert_fail
("I->capturesThis()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 547, __PRETTY_FUNCTION__))
;
548 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
549 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
550 }
551 ++Cnt;
552 ++I;
553 }
554
555 return F;
556}
557
558llvm::Function *
559CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
560 assert(((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function"
) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 562, __PRETTY_FUNCTION__))
561 CapturedStmtInfo &&((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function"
) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 562, __PRETTY_FUNCTION__))
562 "CapturedStmtInfo should be set when generating the captured function")((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function"
) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 562, __PRETTY_FUNCTION__))
;
563 const CapturedDecl *CD = S.getCapturedDecl();
564 // Build the argument list.
565 bool NeedWrapperFunction =
566 getDebugInfo() &&
567 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
568 FunctionArgList Args;
569 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
570 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
571 SmallString<256> Buffer;
572 llvm::raw_svector_ostream Out(Buffer);
573 Out << CapturedStmtInfo->getHelperName();
574 if (NeedWrapperFunction)
575 Out << "_debug__";
576 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
577 Out.str());
578 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
579 VLASizes, CXXThisValue, FO);
580 CodeGenFunction::OMPPrivateScope LocalScope(*this);
581 for (const auto &LocalAddrPair : LocalAddrs) {
582 if (LocalAddrPair.second.first) {
583 LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
584 return LocalAddrPair.second.second;
585 });
586 }
587 }
588 (void)LocalScope.Privatize();
589 for (const auto &VLASizePair : VLASizes)
590 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
591 PGO.assignRegionCounters(GlobalDecl(CD), F);
592 CapturedStmtInfo->EmitBody(*this, CD->getBody());
593 (void)LocalScope.ForceCleanup();
594 FinishFunction(CD->getBodyRBrace());
595 if (!NeedWrapperFunction)
596 return F;
597
598 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
599 /*RegisterCastedArgsOnly=*/true,
600 CapturedStmtInfo->getHelperName());
601 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
602 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
603 Args.clear();
604 LocalAddrs.clear();
605 VLASizes.clear();
606 llvm::Function *WrapperF =
607 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
608 WrapperCGF.CXXThisValue, WrapperFO);
609 llvm::SmallVector<llvm::Value *, 4> CallArgs;
610 for (const auto *Arg : Args) {
611 llvm::Value *CallArg;
612 auto I = LocalAddrs.find(Arg);
613 if (I != LocalAddrs.end()) {
614 LValue LV = WrapperCGF.MakeAddrLValue(
615 I->second.second,
616 I->second.first ? I->second.first->getType() : Arg->getType(),
617 AlignmentSource::Decl);
618 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
619 } else {
620 auto EI = VLASizes.find(Arg);
621 if (EI != VLASizes.end()) {
622 CallArg = EI->second.second;
623 } else {
624 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
625 Arg->getType(),
626 AlignmentSource::Decl);
627 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
628 }
629 }
630 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
631 }
632 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
633 F, CallArgs);
634 WrapperCGF.FinishFunction();
635 return WrapperF;
636}
637
638//===----------------------------------------------------------------------===//
639// OpenMP Directive Emission
640//===----------------------------------------------------------------------===//
641void CodeGenFunction::EmitOMPAggregateAssign(
642 Address DestAddr, Address SrcAddr, QualType OriginalType,
643 const llvm::function_ref<void(Address, Address)> CopyGen) {
644 // Perform element-by-element initialization.
645 QualType ElementTy;
646
647 // Drill down to the base element type on both arrays.
648 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
649 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
650 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
651
652 llvm::Value *SrcBegin = SrcAddr.getPointer();
653 llvm::Value *DestBegin = DestAddr.getPointer();
654 // Cast from pointer to array type to pointer to single element.
655 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
656 // The basic structure here is a while-do loop.
657 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
658 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
659 llvm::Value *IsEmpty =
660 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
661 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
662
663 // Enter the loop body, making that address the current address.
664 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
665 EmitBlock(BodyBB);
666
667 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
668
669 llvm::PHINode *SrcElementPHI =
670 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
671 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
672 Address SrcElementCurrent =
673 Address(SrcElementPHI,
674 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
675
676 llvm::PHINode *DestElementPHI =
677 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
678 DestElementPHI->addIncoming(DestBegin, EntryBB);
679 Address DestElementCurrent =
680 Address(DestElementPHI,
681 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
682
683 // Emit copy.
684 CopyGen(DestElementCurrent, SrcElementCurrent);
685
686 // Shift the address forward by one element.
687 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
688 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
689 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
690 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
691 // Check whether we've reached the end.
692 llvm::Value *Done =
693 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
694 Builder.CreateCondBr(Done, DoneBB, BodyBB);
695 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
696 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
697
698 // Done.
699 EmitBlock(DoneBB, /*IsFinished=*/true);
700}
701
702void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
703 Address SrcAddr, const VarDecl *DestVD,
704 const VarDecl *SrcVD, const Expr *Copy) {
705 if (OriginalType->isArrayType()) {
706 const auto *BO = dyn_cast<BinaryOperator>(Copy);
707 if (BO && BO->getOpcode() == BO_Assign) {
708 // Perform simple memcpy for simple copying.
709 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
710 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
711 EmitAggregateAssign(Dest, Src, OriginalType);
712 } else {
713 // For arrays with complex element types perform element by element
714 // copying.
715 EmitOMPAggregateAssign(
716 DestAddr, SrcAddr, OriginalType,
717 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
718 // Working with the single array element, so have to remap
719 // destination and source variables to corresponding array
720 // elements.
721 CodeGenFunction::OMPPrivateScope Remap(*this);
722 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
723 Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
724 (void)Remap.Privatize();
725 EmitIgnoredExpr(Copy);
726 });
727 }
728 } else {
729 // Remap pseudo source variable to private copy.
730 CodeGenFunction::OMPPrivateScope Remap(*this);
731 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
732 Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
733 (void)Remap.Privatize();
734 // Emit copying of the whole variable.
735 EmitIgnoredExpr(Copy);
736 }
737}
738
739bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
740 OMPPrivateScope &PrivateScope) {
741 if (!HaveInsertPoint())
742 return false;
743 bool DeviceConstTarget =
744 getLangOpts().OpenMPIsDevice &&
745 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
746 bool FirstprivateIsLastprivate = false;
747 llvm::DenseSet<const VarDecl *> Lastprivates;
748 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
749 for (const auto *D : C->varlists())
750 Lastprivates.insert(
751 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
752 }
753 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
754 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
755 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
756 // Force emission of the firstprivate copy if the directive does not emit
757 // outlined function, like omp for, omp simd, omp distribute etc.
758 bool MustEmitFirstprivateCopy =
759 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
760 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
761 auto IRef = C->varlist_begin();
762 auto InitsRef = C->inits().begin();
763 for (const Expr *IInit : C->private_copies()) {
764 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
765 bool ThisFirstprivateIsLastprivate =
766 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
767 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
768 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
769 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
770 !FD->getType()->isReferenceType() &&
771 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
772 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
773 ++IRef;
774 ++InitsRef;
775 continue;
776 }
777 // Do not emit copy for firstprivate constant variables in target regions,
778 // captured by reference.
779 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
780 FD && FD->getType()->isReferenceType() &&
781 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
782 (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
783 OrigVD);
784 ++IRef;
785 ++InitsRef;
786 continue;
787 }
788 FirstprivateIsLastprivate =
789 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
790 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
791 const auto *VDInit =
792 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
793 bool IsRegistered;
794 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
795 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
796 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
797 LValue OriginalLVal;
798 if (!FD) {
799 // Check if the firstprivate variable is just a constant value.
800 ConstantEmission CE = tryEmitAsConstant(&DRE);
801 if (CE && !CE.isReference()) {
802 // Constant value, no need to create a copy.
803 ++IRef;
804 ++InitsRef;
805 continue;
806 }
807 if (CE && CE.isReference()) {
808 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
809 } else {
810 assert(!CE && "Expected non-constant firstprivate.")((!CE && "Expected non-constant firstprivate.") ? static_cast
<void> (0) : __assert_fail ("!CE && \"Expected non-constant firstprivate.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 810, __PRETTY_FUNCTION__))
;
811 OriginalLVal = EmitLValue(&DRE);
812 }
813 } else {
814 OriginalLVal = EmitLValue(&DRE);
815 }
816 QualType Type = VD->getType();
817 if (Type->isArrayType()) {
818 // Emit VarDecl with copy init for arrays.
819 // Get the address of the original variable captured in current
820 // captured region.
821 IsRegistered = PrivateScope.addPrivate(
822 OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
823 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
824 const Expr *Init = VD->getInit();
825 if (!isa<CXXConstructExpr>(Init) ||
826 isTrivialInitializer(Init)) {
827 // Perform simple memcpy.
828 LValue Dest =
829 MakeAddrLValue(Emission.getAllocatedAddress(), Type);
830 EmitAggregateAssign(Dest, OriginalLVal, Type);
831 } else {
832 EmitOMPAggregateAssign(
833 Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
834 Type,
835 [this, VDInit, Init](Address DestElement,
836 Address SrcElement) {
837 // Clean up any temporaries needed by the
838 // initialization.
839 RunCleanupsScope InitScope(*this);
840 // Emit initialization for single element.
841 setAddrOfLocalVar(VDInit, SrcElement);
842 EmitAnyExprToMem(Init, DestElement,
843 Init->getType().getQualifiers(),
844 /*IsInitializer*/ false);
845 LocalDeclMap.erase(VDInit);
846 });
847 }
848 EmitAutoVarCleanups(Emission);
849 return Emission.getAllocatedAddress();
850 });
851 } else {
852 Address OriginalAddr = OriginalLVal.getAddress();
853 IsRegistered = PrivateScope.addPrivate(
854 OrigVD, [this, VDInit, OriginalAddr, VD]() {
855 // Emit private VarDecl with copy init.
856 // Remap temp VDInit variable to the address of the original
857 // variable (for proper handling of captured global variables).
858 setAddrOfLocalVar(VDInit, OriginalAddr);
859 EmitDecl(*VD);
860 LocalDeclMap.erase(VDInit);
861 return GetAddrOfLocalVar(VD);
862 });
863 }
864 assert(IsRegistered &&((IsRegistered && "firstprivate var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 865, __PRETTY_FUNCTION__))
865 "firstprivate var already registered as private")((IsRegistered && "firstprivate var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 865, __PRETTY_FUNCTION__))
;
866 // Silence the warning about unused variable.
867 (void)IsRegistered;
868 }
869 ++IRef;
870 ++InitsRef;
871 }
872 }
873 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
874}
875
876void CodeGenFunction::EmitOMPPrivateClause(
877 const OMPExecutableDirective &D,
878 CodeGenFunction::OMPPrivateScope &PrivateScope) {
879 if (!HaveInsertPoint())
880 return;
881 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
882 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
883 auto IRef = C->varlist_begin();
884 for (const Expr *IInit : C->private_copies()) {
885 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
886 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
887 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
888 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
889 // Emit private VarDecl with copy init.
890 EmitDecl(*VD);
891 return GetAddrOfLocalVar(VD);
892 });
893 assert(IsRegistered && "private var already registered as private")((IsRegistered && "private var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"private var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 893, __PRETTY_FUNCTION__))
;
894 // Silence the warning about unused variable.
895 (void)IsRegistered;
896 }
897 ++IRef;
898 }
899 }
900}
901
902bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
903 if (!HaveInsertPoint())
904 return false;
905 // threadprivate_var1 = master_threadprivate_var1;
906 // operator=(threadprivate_var2, master_threadprivate_var2);
907 // ...
908 // __kmpc_barrier(&loc, global_tid);
909 llvm::DenseSet<const VarDecl *> CopiedVars;
910 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
911 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
912 auto IRef = C->varlist_begin();
913 auto ISrcRef = C->source_exprs().begin();
914 auto IDestRef = C->destination_exprs().begin();
915 for (const Expr *AssignOp : C->assignment_ops()) {
916 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
917 QualType Type = VD->getType();
918 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
919 // Get the address of the master variable. If we are emitting code with
920 // TLS support, the address is passed from the master as field in the
921 // captured declaration.
922 Address MasterAddr = Address::invalid();
923 if (getLangOpts().OpenMPUseTLS &&
924 getContext().getTargetInfo().isTLSSupported()) {
925 assert(CapturedStmtInfo->lookup(VD) &&((CapturedStmtInfo->lookup(VD) && "Copyin threadprivates should have been captured!"
) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo->lookup(VD) && \"Copyin threadprivates should have been captured!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 926, __PRETTY_FUNCTION__))
926 "Copyin threadprivates should have been captured!")((CapturedStmtInfo->lookup(VD) && "Copyin threadprivates should have been captured!"
) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo->lookup(VD) && \"Copyin threadprivates should have been captured!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 926, __PRETTY_FUNCTION__))
;
927 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
928 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
929 MasterAddr = EmitLValue(&DRE).getAddress();
930 LocalDeclMap.erase(VD);
931 } else {
932 MasterAddr =
933 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
934 : CGM.GetAddrOfGlobal(VD),
935 getContext().getDeclAlign(VD));
936 }
937 // Get the address of the threadprivate variable.
938 Address PrivateAddr = EmitLValue(*IRef).getAddress();
939 if (CopiedVars.size() == 1) {
940 // At first check if current thread is a master thread. If it is, no
941 // need to copy data.
942 CopyBegin = createBasicBlock("copyin.not.master");
943 CopyEnd = createBasicBlock("copyin.not.master.end");
944 Builder.CreateCondBr(
945 Builder.CreateICmpNE(
946 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
947 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
948 CGM.IntPtrTy)),
949 CopyBegin, CopyEnd);
950 EmitBlock(CopyBegin);
951 }
952 const auto *SrcVD =
953 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
954 const auto *DestVD =
955 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
956 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
957 }
958 ++IRef;
959 ++ISrcRef;
960 ++IDestRef;
961 }
962 }
963 if (CopyEnd) {
964 // Exit out of copying procedure for non-master thread.
965 EmitBlock(CopyEnd, /*IsFinished=*/true);
966 return true;
967 }
968 return false;
969}
970
971bool CodeGenFunction::EmitOMPLastprivateClauseInit(
972 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
973 if (!HaveInsertPoint())
974 return false;
975 bool HasAtLeastOneLastprivate = false;
976 llvm::DenseSet<const VarDecl *> SIMDLCVs;
977 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
978 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
979 for (const Expr *C : LoopDirective->counters()) {
980 SIMDLCVs.insert(
981 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
982 }
983 }
984 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
985 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
986 HasAtLeastOneLastprivate = true;
987 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
988 !getLangOpts().OpenMPSimd)
989 break;
990 auto IRef = C->varlist_begin();
991 auto IDestRef = C->destination_exprs().begin();
992 for (const Expr *IInit : C->private_copies()) {
993 // Keep the address of the original variable for future update at the end
994 // of the loop.
995 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
996 // Taskloops do not require additional initialization, it is done in
997 // runtime support library.
998 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
999 const auto *DestVD =
1000 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1001 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
1002 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1003 /*RefersToEnclosingVariableOrCapture=*/
1004 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1005 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1006 return EmitLValue(&DRE).getAddress();
1007 });
1008 // Check if the variable is also a firstprivate: in this case IInit is
1009 // not generated. Initialization of this variable will happen in codegen
1010 // for 'firstprivate' clause.
1011 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1012 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1013 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
1014 // Emit private VarDecl with copy init.
1015 EmitDecl(*VD);
1016 return GetAddrOfLocalVar(VD);
1017 });
1018 assert(IsRegistered &&((IsRegistered && "lastprivate var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"lastprivate var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1019, __PRETTY_FUNCTION__))
1019 "lastprivate var already registered as private")((IsRegistered && "lastprivate var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"lastprivate var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1019, __PRETTY_FUNCTION__))
;
1020 (void)IsRegistered;
1021 }
1022 }
1023 ++IRef;
1024 ++IDestRef;
1025 }
1026 }
1027 return HasAtLeastOneLastprivate;
1028}
1029
1030void CodeGenFunction::EmitOMPLastprivateClauseFinal(
1031 const OMPExecutableDirective &D, bool NoFinals,
1032 llvm::Value *IsLastIterCond) {
1033 if (!HaveInsertPoint())
1034 return;
1035 // Emit following code:
1036 // if (<IsLastIterCond>) {
1037 // orig_var1 = private_orig_var1;
1038 // ...
1039 // orig_varn = private_orig_varn;
1040 // }
1041 llvm::BasicBlock *ThenBB = nullptr;
1042 llvm::BasicBlock *DoneBB = nullptr;
1043 if (IsLastIterCond) {
1044 ThenBB = createBasicBlock(".omp.lastprivate.then");
1045 DoneBB = createBasicBlock(".omp.lastprivate.done");
1046 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1047 EmitBlock(ThenBB);
1048 }
1049 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1050 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1051 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1052 auto IC = LoopDirective->counters().begin();
1053 for (const Expr *F : LoopDirective->finals()) {
1054 const auto *D =
1055 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1056 if (NoFinals)
1057 AlreadyEmittedVars.insert(D);
1058 else
1059 LoopCountersAndUpdates[D] = F;
1060 ++IC;
1061 }
1062 }
1063 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1064 auto IRef = C->varlist_begin();
1065 auto ISrcRef = C->source_exprs().begin();
1066 auto IDestRef = C->destination_exprs().begin();
1067 for (const Expr *AssignOp : C->assignment_ops()) {
1068 const auto *PrivateVD =
1069 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1070 QualType Type = PrivateVD->getType();
1071 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1072 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1073 // If lastprivate variable is a loop control variable for loop-based
1074 // directive, update its value before copyin back to original
1075 // variable.
1076 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1077 EmitIgnoredExpr(FinalExpr);
1078 const auto *SrcVD =
1079 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1080 const auto *DestVD =
1081 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1082 // Get the address of the original variable.
1083 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1084 // Get the address of the private variable.
1085 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1086 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1087 PrivateAddr =
1088 Address(Builder.CreateLoad(PrivateAddr),
1089 getNaturalTypeAlignment(RefTy->getPointeeType()));
1090 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1091 }
1092 ++IRef;
1093 ++ISrcRef;
1094 ++IDestRef;
1095 }
1096 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1097 EmitIgnoredExpr(PostUpdate);
1098 }
1099 if (IsLastIterCond)
1100 EmitBlock(DoneBB, /*IsFinished=*/true);
1101}
1102
1103void CodeGenFunction::EmitOMPReductionClauseInit(
1104 const OMPExecutableDirective &D,
1105 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1106 if (!HaveInsertPoint())
1107 return;
1108 SmallVector<const Expr *, 4> Shareds;
1109 SmallVector<const Expr *, 4> Privates;
1110 SmallVector<const Expr *, 4> ReductionOps;
1111 SmallVector<const Expr *, 4> LHSs;
1112 SmallVector<const Expr *, 4> RHSs;
1113 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1114 auto IPriv = C->privates().begin();
1115 auto IRed = C->reduction_ops().begin();
1116 auto ILHS = C->lhs_exprs().begin();
1117 auto IRHS = C->rhs_exprs().begin();
1118 for (const Expr *Ref : C->varlists()) {
1119 Shareds.emplace_back(Ref);
1120 Privates.emplace_back(*IPriv);
1121 ReductionOps.emplace_back(*IRed);
1122 LHSs.emplace_back(*ILHS);
1123 RHSs.emplace_back(*IRHS);
1124 std::advance(IPriv, 1);
1125 std::advance(IRed, 1);
1126 std::advance(ILHS, 1);
1127 std::advance(IRHS, 1);
1128 }
1129 }
1130 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1131 unsigned Count = 0;
1132 auto ILHS = LHSs.begin();
1133 auto IRHS = RHSs.begin();
1134 auto IPriv = Privates.begin();
1135 for (const Expr *IRef : Shareds) {
1136 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1137 // Emit private VarDecl with reduction init.
1138 RedCG.emitSharedLValue(*this, Count);
1139 RedCG.emitAggregateType(*this, Count);
1140 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1141 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1142 RedCG.getSharedLValue(Count),
1143 [&Emission](CodeGenFunction &CGF) {
1144 CGF.EmitAutoVarInit(Emission);
1145 return true;
1146 });
1147 EmitAutoVarCleanups(Emission);
1148 Address BaseAddr = RedCG.adjustPrivateAddress(
1149 *this, Count, Emission.getAllocatedAddress());
1150 bool IsRegistered = PrivateScope.addPrivate(
1151 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1152 assert(IsRegistered && "private var already registered as private")((IsRegistered && "private var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"private var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1152, __PRETTY_FUNCTION__))
;
1153 // Silence the warning about unused variable.
1154 (void)IsRegistered;
1155
1156 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1157 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1158 QualType Type = PrivateVD->getType();
1159 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1160 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1161 // Store the address of the original variable associated with the LHS
1162 // implicit variable.
1163 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1164 return RedCG.getSharedLValue(Count).getAddress();
1165 });
1166 PrivateScope.addPrivate(
1167 RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1168 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1169 isa<ArraySubscriptExpr>(IRef)) {
1170 // Store the address of the original variable associated with the LHS
1171 // implicit variable.
1172 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1173 return RedCG.getSharedLValue(Count).getAddress();
1174 });
1175 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1176 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1177 ConvertTypeForMem(RHSVD->getType()),
1178 "rhs.begin");
1179 });
1180 } else {
1181 QualType Type = PrivateVD->getType();
1182 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1183 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1184 // Store the address of the original variable associated with the LHS
1185 // implicit variable.
1186 if (IsArray) {
1187 OriginalAddr = Builder.CreateElementBitCast(
1188 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1189 }
1190 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1191 PrivateScope.addPrivate(
1192 RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1193 return IsArray
1194 ? Builder.CreateElementBitCast(
1195 GetAddrOfLocalVar(PrivateVD),
1196 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1197 : GetAddrOfLocalVar(PrivateVD);
1198 });
1199 }
1200 ++ILHS;
1201 ++IRHS;
1202 ++IPriv;
1203 ++Count;
1204 }
1205}
1206
1207void CodeGenFunction::EmitOMPReductionClauseFinal(
1208 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1209 if (!HaveInsertPoint())
1210 return;
1211 llvm::SmallVector<const Expr *, 8> Privates;
1212 llvm::SmallVector<const Expr *, 8> LHSExprs;
1213 llvm::SmallVector<const Expr *, 8> RHSExprs;
1214 llvm::SmallVector<const Expr *, 8> ReductionOps;
1215 bool HasAtLeastOneReduction = false;
1216 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1217 HasAtLeastOneReduction = true;
1218 Privates.append(C->privates().begin(), C->privates().end());
1219 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1220 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1221 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1222 }
1223 if (HasAtLeastOneReduction) {
1224 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1225 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1226 ReductionKind == OMPD_simd;
1227 bool SimpleReduction = ReductionKind == OMPD_simd;
1228 // Emit nowait reduction if nowait clause is present or directive is a
1229 // parallel directive (it always has implicit barrier).
1230 CGM.getOpenMPRuntime().emitReduction(
1231 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1232 {WithNowait, SimpleReduction, ReductionKind});
1233 }
1234}
1235
1236static void emitPostUpdateForReductionClause(
1237 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1238 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1239 if (!CGF.HaveInsertPoint())
1240 return;
1241 llvm::BasicBlock *DoneBB = nullptr;
1242 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1243 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1244 if (!DoneBB) {
1245 if (llvm::Value *Cond = CondGen(CGF)) {
1246 // If the first post-update expression is found, emit conditional
1247 // block if it was requested.
1248 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1249 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1250 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1251 CGF.EmitBlock(ThenBB);
1252 }
1253 }
1254 CGF.EmitIgnoredExpr(PostUpdate);
1255 }
1256 }
1257 if (DoneBB)
1258 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1259}
1260
1261namespace {
1262/// Codegen lambda for appending distribute lower and upper bounds to outlined
1263/// parallel function. This is necessary for combined constructs such as
1264/// 'distribute parallel for'
1265typedef llvm::function_ref<void(CodeGenFunction &,
1266 const OMPExecutableDirective &,
1267 llvm::SmallVectorImpl<llvm::Value *> &)>
1268 CodeGenBoundParametersTy;
1269} // anonymous namespace
1270
1271static void emitCommonOMPParallelDirective(
1272 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1273 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1274 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1275 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1276 llvm::Function *OutlinedFn =
1277 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1278 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1279 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1280 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1281 llvm::Value *NumThreads =
1282 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1283 /*IgnoreResultAssign=*/true);
1284 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1285 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1286 }
1287 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1288 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1289 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1290 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1291 }
1292 const Expr *IfCond = nullptr;
1293 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1294 if (C->getNameModifier() == OMPD_unknown ||
1295 C->getNameModifier() == OMPD_parallel) {
1296 IfCond = C->getCondition();
1297 break;
1298 }
1299 }
1300
1301 OMPParallelScope Scope(CGF, S);
1302 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1303 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1304 // lower and upper bounds with the pragma 'for' chunking mechanism.
1305 // The following lambda takes care of appending the lower and upper bound
1306 // parameters when necessary
1307 CodeGenBoundParameters(CGF, S, CapturedVars);
1308 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1309 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1310 CapturedVars, IfCond);
1311}
1312
1313static void emitEmptyBoundParameters(CodeGenFunction &,
1314 const OMPExecutableDirective &,
1315 llvm::SmallVectorImpl<llvm::Value *> &) {}
1316
1317void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1318 // Emit parallel region as a standalone region.
1319 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1320 Action.Enter(CGF);
1321 OMPPrivateScope PrivateScope(CGF);
1322 bool Copyins = CGF.EmitOMPCopyinClause(S);
1323 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1324 if (Copyins) {
1325 // Emit implicit barrier to synchronize threads and avoid data races on
1326 // propagation master's thread values of threadprivate variables to local
1327 // instances of that variables of all other implicit threads.
1328 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1329 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1330 /*ForceSimpleCall=*/true);
1331 }
1332 CGF.EmitOMPPrivateClause(S, PrivateScope);
1333 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1334 (void)PrivateScope.Privatize();
1335 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1336 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1337 };
1338 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1339 emitEmptyBoundParameters);
1340 emitPostUpdateForReductionClause(*this, S,
1341 [](CodeGenFunction &) { return nullptr; });
1342}
1343
1344static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1345 int MaxLevel, int Level = 0) {
1346 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.")((Level < MaxLevel && "Too deep lookup during loop body codegen."
) ? static_cast<void> (0) : __assert_fail ("Level < MaxLevel && \"Too deep lookup during loop body codegen.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1346, __PRETTY_FUNCTION__))
;
1347 const Stmt *SimplifiedS = S->IgnoreContainers();
1348 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1349 PrettyStackTraceLoc CrashInfo(
1350 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1351 "LLVM IR generation of compound statement ('{}')");
1352
1353 // Keep track of the current cleanup stack depth, including debug scopes.
1354 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1355 for (const Stmt *CurStmt : CS->body())
1356 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1357 return;
1358 }
1359 if (SimplifiedS == NextLoop) {
1360 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1361 S = For->getBody();
1362 } else {
1363 assert(isa<CXXForRangeStmt>(SimplifiedS) &&((isa<CXXForRangeStmt>(SimplifiedS) && "Expected canonical for loop or range-based for loop."
) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(SimplifiedS) && \"Expected canonical for loop or range-based for loop.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1364, __PRETTY_FUNCTION__))
1364 "Expected canonical for loop or range-based for loop.")((isa<CXXForRangeStmt>(SimplifiedS) && "Expected canonical for loop or range-based for loop."
) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(SimplifiedS) && \"Expected canonical for loop or range-based for loop.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1364, __PRETTY_FUNCTION__))
;
1365 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1366 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1367 S = CXXFor->getBody();
1368 }
1369 if (Level + 1 < MaxLevel) {
1370 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
1371 S, /*TryImperfectlyNestedLoops=*/true);
1372 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1373 return;
1374 }
1375 }
1376 CGF.EmitStmt(S);
1377}
1378
1379void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1380 JumpDest LoopExit) {
1381 RunCleanupsScope BodyScope(*this);
1382 // Update counters values on current iteration.
1383 for (const Expr *UE : D.updates())
1384 EmitIgnoredExpr(UE);
1385 // Update the linear variables.
1386 // In distribute directives only loop counters may be marked as linear, no
1387 // need to generate the code for them.
1388 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1389 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1390 for (const Expr *UE : C->updates())
1391 EmitIgnoredExpr(UE);
1392 }
1393 }
1394
1395 // On a continue in the body, jump to the end.
1396 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1397 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1398 for (const Expr *E : D.finals_conditions()) {
1399 if (!E)
1400 continue;
1401 // Check that loop counter in non-rectangular nest fits into the iteration
1402 // space.
1403 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1404 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1405 getProfileCount(D.getBody()));
1406 EmitBlock(NextBB);
1407 }
1408 // Emit loop variables for C++ range loops.
1409 const Stmt *Body =
1410 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
1411 // Emit loop body.
1412 emitBody(*this, Body,
1413 OMPLoopDirective::tryToFindNextInnerLoop(
1414 Body, /*TryImperfectlyNestedLoops=*/true),
1415 D.getCollapsedNumber());
1416
1417 // The end (updates/cleanups).
1418 EmitBlock(Continue.getBlock());
1419 BreakContinueStack.pop_back();
1420}
1421
1422void CodeGenFunction::EmitOMPInnerLoop(
1423 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1424 const Expr *IncExpr,
1425 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1426 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1427 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1428
1429 // Start the loop with a block that tests the condition.
1430 auto CondBlock = createBasicBlock("omp.inner.for.cond");
1431 EmitBlock(CondBlock);
1432 const SourceRange R = S.getSourceRange();
1433 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1434 SourceLocToDebugLoc(R.getEnd()));
1435
1436 // If there are any cleanups between here and the loop-exit scope,
1437 // create a block to stage a loop exit along.
1438 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1439 if (RequiresCleanup)
1440 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1441
1442 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1443
1444 // Emit condition.
1445 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1446 if (ExitBlock != LoopExit.getBlock()) {
1447 EmitBlock(ExitBlock);
1448 EmitBranchThroughCleanup(LoopExit);
1449 }
1450
1451 EmitBlock(LoopBody);
1452 incrementProfileCounter(&S);
1453
1454 // Create a block for the increment.
1455 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1456 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1457
1458 BodyGen(*this);
1459
1460 // Emit "IV = IV + 1" and a back-edge to the condition block.
1461 EmitBlock(Continue.getBlock());
1462 EmitIgnoredExpr(IncExpr);
1463 PostIncGen(*this);
1464 BreakContinueStack.pop_back();
1465 EmitBranch(CondBlock);
1466 LoopStack.pop();
1467 // Emit the fall-through block.
1468 EmitBlock(LoopExit.getBlock());
1469}
1470
1471bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1472 if (!HaveInsertPoint())
1473 return false;
1474 // Emit inits for the linear variables.
1475 bool HasLinears = false;
1476 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1477 for (const Expr *Init : C->inits()) {
1478 HasLinears = true;
1479 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1480 if (const auto *Ref =
1481 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1482 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1483 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1484 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1485 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1486 VD->getInit()->getType(), VK_LValue,
1487 VD->getInit()->getExprLoc());
1488 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1489 VD->getType()),
1490 /*capturedByInit=*/false);
1491 EmitAutoVarCleanups(Emission);
1492 } else {
1493 EmitVarDecl(*VD);
1494 }
1495 }
1496 // Emit the linear steps for the linear clauses.
1497 // If a step is not constant, it is pre-calculated before the loop.
1498 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1499 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1500 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1501 // Emit calculation of the linear step.
1502 EmitIgnoredExpr(CS);
1503 }
1504 }
1505 return HasLinears;
1506}
1507
1508void CodeGenFunction::EmitOMPLinearClauseFinal(
1509 const OMPLoopDirective &D,
1510 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1511 if (!HaveInsertPoint())
1512 return;
1513 llvm::BasicBlock *DoneBB = nullptr;
1514 // Emit the final values of the linear variables.
1515 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1516 auto IC = C->varlist_begin();
1517 for (const Expr *F : C->finals()) {
1518 if (!DoneBB) {
1519 if (llvm::Value *Cond = CondGen(*this)) {
1520 // If the first post-update expression is found, emit conditional
1521 // block if it was requested.
1522 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1523 DoneBB = createBasicBlock(".omp.linear.pu.done");
1524 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1525 EmitBlock(ThenBB);
1526 }
1527 }
1528 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1529 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1530 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1531 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1532 Address OrigAddr = EmitLValue(&DRE).getAddress();
1533 CodeGenFunction::OMPPrivateScope VarScope(*this);
1534 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1535 (void)VarScope.Privatize();
1536 EmitIgnoredExpr(F);
1537 ++IC;
1538 }
1539 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1540 EmitIgnoredExpr(PostUpdate);
1541 }
1542 if (DoneBB)
1543 EmitBlock(DoneBB, /*IsFinished=*/true);
1544}
1545
1546static void emitAlignedClause(CodeGenFunction &CGF,
1547 const OMPExecutableDirective &D) {
1548 if (!CGF.HaveInsertPoint())
1549 return;
1550 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1551 llvm::APInt ClauseAlignment(64, 0);
1552 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1553 auto *AlignmentCI =
1554 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1555 ClauseAlignment = AlignmentCI->getValue();
1556 }
1557 for (const Expr *E : Clause->varlists()) {
1558 llvm::APInt Alignment(ClauseAlignment);
1559 if (Alignment == 0) {
1560 // OpenMP [2.8.1, Description]
1561 // If no optional parameter is specified, implementation-defined default
1562 // alignments for SIMD instructions on the target platforms are assumed.
1563 Alignment =
1564 CGF.getContext()
1565 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1566 E->getType()->getPointeeType()))
1567 .getQuantity();
1568 }
1569 assert((Alignment == 0 || Alignment.isPowerOf2()) &&(((Alignment == 0 || Alignment.isPowerOf2()) && "alignment is not power of 2"
) ? static_cast<void> (0) : __assert_fail ("(Alignment == 0 || Alignment.isPowerOf2()) && \"alignment is not power of 2\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1570, __PRETTY_FUNCTION__))
1570 "alignment is not power of 2")(((Alignment == 0 || Alignment.isPowerOf2()) && "alignment is not power of 2"
) ? static_cast<void> (0) : __assert_fail ("(Alignment == 0 || Alignment.isPowerOf2()) && \"alignment is not power of 2\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1570, __PRETTY_FUNCTION__))
;
1571 if (Alignment != 0) {
1572 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1573 CGF.EmitAlignmentAssumption(
1574 PtrValue, E, /*No second loc needed*/ SourceLocation(),
1575 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
1576 }
1577 }
1578 }
1579}
1580
1581void CodeGenFunction::EmitOMPPrivateLoopCounters(
1582 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1583 if (!HaveInsertPoint())
1584 return;
1585 auto I = S.private_counters().begin();
1586 for (const Expr *E : S.counters()) {
1587 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1588 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1589 // Emit var without initialization.
1590 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1591 EmitAutoVarCleanups(VarEmission);
1592 LocalDeclMap.erase(PrivateVD);
1593 (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1594 return VarEmission.getAllocatedAddress();
1595 });
1596 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1597 VD->hasGlobalStorage()) {
1598 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1599 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1600 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1601 E->getType(), VK_LValue, E->getExprLoc());
1602 return EmitLValue(&DRE).getAddress();
1603 });
1604 } else {
1605 (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1606 return VarEmission.getAllocatedAddress();
1607 });
1608 }
1609 ++I;
1610 }
1611 // Privatize extra loop counters used in loops for ordered(n) clauses.
1612 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1613 if (!C->getNumForLoops())
1614 continue;
1615 for (unsigned I = S.getCollapsedNumber(),
1616 E = C->getLoopNumIterations().size();
1617 I < E; ++I) {
1618 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1619 const auto *VD = cast<VarDecl>(DRE->getDecl());
1620 // Override only those variables that can be captured to avoid re-emission
1621 // of the variables declared within the loops.
1622 if (DRE->refersToEnclosingVariableOrCapture()) {
1623 (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1624 return CreateMemTemp(DRE->getType(), VD->getName());
1625 });
1626 }
1627 }
1628 }
1629}
1630
1631static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1632 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1633 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1634 if (!CGF.HaveInsertPoint())
1635 return;
1636 {
1637 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1638 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1639 (void)PreCondScope.Privatize();
1640 // Get initial values of real counters.
1641 for (const Expr *I : S.inits()) {
1642 CGF.EmitIgnoredExpr(I);
1643 }
1644 }
1645 // Create temp loop control variables with their init values to support
1646 // non-rectangular loops.
1647 CodeGenFunction::OMPMapVars PreCondVars;
1648 for (const Expr * E: S.dependent_counters()) {
1649 if (!E)
1650 continue;
1651 assert(!E->getType().getNonReferenceType()->isRecordType() &&((!E->getType().getNonReferenceType()->isRecordType() &&
"dependent counter must not be an iterator.") ? static_cast<
void> (0) : __assert_fail ("!E->getType().getNonReferenceType()->isRecordType() && \"dependent counter must not be an iterator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1652, __PRETTY_FUNCTION__))
1652 "dependent counter must not be an iterator.")((!E->getType().getNonReferenceType()->isRecordType() &&
"dependent counter must not be an iterator.") ? static_cast<
void> (0) : __assert_fail ("!E->getType().getNonReferenceType()->isRecordType() && \"dependent counter must not be an iterator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1652, __PRETTY_FUNCTION__))
;
1653 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1654 Address CounterAddr =
1655 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
1656 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
1657 }
1658 (void)PreCondVars.apply(CGF);
1659 for (const Expr *E : S.dependent_inits()) {
1660 if (!E)
1661 continue;
1662 CGF.EmitIgnoredExpr(E);
1663 }
1664 // Check that loop is executed at least one time.
1665 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1666 PreCondVars.restore(CGF);
1667}
1668
1669void CodeGenFunction::EmitOMPLinearClause(
1670 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1671 if (!HaveInsertPoint())
1672 return;
1673 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1674 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1675 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1676 for (const Expr *C : LoopDirective->counters()) {
1677 SIMDLCVs.insert(
1678 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1679 }
1680 }
1681 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1682 auto CurPrivate = C->privates().begin();
1683 for (const Expr *E : C->varlists()) {
1684 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1685 const auto *PrivateVD =
1686 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1687 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1688 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1689 // Emit private VarDecl with copy init.
1690 EmitVarDecl(*PrivateVD);
1691 return GetAddrOfLocalVar(PrivateVD);
1692 });
1693 assert(IsRegistered && "linear var already registered as private")((IsRegistered && "linear var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"linear var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1693, __PRETTY_FUNCTION__))
;
1694 // Silence the warning about unused variable.
1695 (void)IsRegistered;
1696 } else {
1697 EmitVarDecl(*PrivateVD);
1698 }
1699 ++CurPrivate;
1700 }
1701 }
1702}
1703
1704static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1705 const OMPExecutableDirective &D,
1706 bool IsMonotonic) {
1707 if (!CGF.HaveInsertPoint())
1708 return;
1709 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1710 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1711 /*ignoreResult=*/true);
1712 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1713 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1714 // In presence of finite 'safelen', it may be unsafe to mark all
1715 // the memory instructions parallel, because loop-carried
1716 // dependences of 'safelen' iterations are possible.
1717 if (!IsMonotonic)
1718 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1719 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1720 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1721 /*ignoreResult=*/true);
1722 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1723 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1724 // In presence of finite 'safelen', it may be unsafe to mark all
1725 // the memory instructions parallel, because loop-carried
1726 // dependences of 'safelen' iterations are possible.
1727 CGF.LoopStack.setParallel(/*Enable=*/false);
1728 }
1729}
1730
1731void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1732 bool IsMonotonic) {
1733 // Walk clauses and process safelen/lastprivate.
1734 LoopStack.setParallel(!IsMonotonic);
1735 LoopStack.setVectorizeEnable();
1736 emitSimdlenSafelenClause(*this, D, IsMonotonic);
1737}
1738
1739void CodeGenFunction::EmitOMPSimdFinal(
1740 const OMPLoopDirective &D,
1741 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1742 if (!HaveInsertPoint())
1743 return;
1744 llvm::BasicBlock *DoneBB = nullptr;
1745 auto IC = D.counters().begin();
1746 auto IPC = D.private_counters().begin();
1747 for (const Expr *F : D.finals()) {
1748 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1749 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1750 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1751 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1752 OrigVD->hasGlobalStorage() || CED) {
1753 if (!DoneBB) {
1754 if (llvm::Value *Cond = CondGen(*this)) {
1755 // If the first post-update expression is found, emit conditional
1756 // block if it was requested.
1757 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1758 DoneBB = createBasicBlock(".omp.final.done");
1759 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1760 EmitBlock(ThenBB);
1761 }
1762 }
1763 Address OrigAddr = Address::invalid();
1764 if (CED) {
1765 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1766 } else {
1767 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1768 /*RefersToEnclosingVariableOrCapture=*/false,
1769 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1770 OrigAddr = EmitLValue(&DRE).getAddress();
1771 }
1772 OMPPrivateScope VarScope(*this);
1773 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1774 (void)VarScope.Privatize();
1775 EmitIgnoredExpr(F);
1776 }
1777 ++IC;
1778 ++IPC;
1779 }
1780 if (DoneBB)
1781 EmitBlock(DoneBB, /*IsFinished=*/true);
1782}
1783
1784static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1785 const OMPLoopDirective &S,
1786 CodeGenFunction::JumpDest LoopExit) {
1787 CGF.EmitOMPLoopBody(S, LoopExit);
1788 CGF.EmitStopPoint(&S);
1789}
1790
1791/// Emit a helper variable and return corresponding lvalue.
1792static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1793 const DeclRefExpr *Helper) {
1794 auto VDecl = cast<VarDecl>(Helper->getDecl());
1795 CGF.EmitVarDecl(*VDecl);
1796 return CGF.EmitLValue(Helper);
1797}
1798
1799static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1800 PrePostActionTy &Action) {
1801 Action.Enter(CGF);
1802 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&((isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive"
) ? static_cast<void> (0) : __assert_fail ("isOpenMPSimdDirective(S.getDirectiveKind()) && \"Expected simd directive\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1803, __PRETTY_FUNCTION__))
1803 "Expected simd directive")((isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive"
) ? static_cast<void> (0) : __assert_fail ("isOpenMPSimdDirective(S.getDirectiveKind()) && \"Expected simd directive\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 1803, __PRETTY_FUNCTION__))
;
1804 OMPLoopScope PreInitScope(CGF, S);
1805 // if (PreCond) {
1806 // for (IV in 0..LastIteration) BODY;
1807 // <Final counter/linear vars updates>;
1808 // }
1809 //
1810 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1811 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
1812 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
1813 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1814 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1815 }
1816
1817 // Emit: if (PreCond) - begin.
1818 // If the condition constant folds and can be elided, avoid emitting the
1819 // whole loop.
1820 bool CondConstant;
1821 llvm::BasicBlock *ContBlock = nullptr;
1822 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1823 if (!CondConstant)
1824 return;
1825 } else {
1826 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1827 ContBlock = CGF.createBasicBlock("simd.if.end");
1828 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1829 CGF.getProfileCount(&S));
1830 CGF.EmitBlock(ThenBlock);
1831 CGF.incrementProfileCounter(&S);
1832 }
1833
1834 // Emit the loop iteration variable.
1835 const Expr *IVExpr = S.getIterationVariable();
1836 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1837 CGF.EmitVarDecl(*IVDecl);
1838 CGF.EmitIgnoredExpr(S.getInit());
1839
1840 // Emit the iterations count variable.
1841 // If it is not a variable, Sema decided to calculate iterations count on
1842 // each iteration (e.g., it is foldable into a constant).
1843 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1844 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1845 // Emit calculation of the iterations count.
1846 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1847 }
1848
1849 CGF.EmitOMPSimdInit(S);
1850
1851 emitAlignedClause(CGF, S);
1852 (void)CGF.EmitOMPLinearClauseInit(S);
1853 {
1854 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1855 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1856 CGF.EmitOMPLinearClause(S, LoopScope);
1857 CGF.EmitOMPPrivateClause(S, LoopScope);
1858 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1859 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1860 (void)LoopScope.Privatize();
1861 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
1862 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1863 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1864 S.getInc(),
1865 [&S](CodeGenFunction &CGF) {
1866 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1867 CGF.EmitStopPoint(&S);
1868 },
1869 [](CodeGenFunction &) {});
1870 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1871 // Emit final copy of the lastprivate variables at the end of loops.
1872 if (HasLastprivateClause)
1873 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1874 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1875 emitPostUpdateForReductionClause(CGF, S,
1876 [](CodeGenFunction &) { return nullptr; });
1877 }
1878 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1879 // Emit: if (PreCond) - end.
1880 if (ContBlock) {
1881 CGF.EmitBranch(ContBlock);
1882 CGF.EmitBlock(ContBlock, true);
1883 }
1884}
1885
1886void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1887 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1888 emitOMPSimdRegion(CGF, S, Action);
1889 };
1890 OMPLexicalScope Scope(*this, S, OMPD_unknown);
1891 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1892}
1893
1894void CodeGenFunction::EmitOMPOuterLoop(
1895 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1896 CodeGenFunction::OMPPrivateScope &LoopScope,
1897 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1898 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1899 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1900 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1901
1902 const Expr *IVExpr = S.getIterationVariable();
1903 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1904 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1905
1906 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1907
1908 // Start the loop with a block that tests the condition.
1909 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1910 EmitBlock(CondBlock);
1911 const SourceRange R = S.getSourceRange();
1912 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1913 SourceLocToDebugLoc(R.getEnd()));
1914
1915 llvm::Value *BoolCondVal = nullptr;
1916 if (!DynamicOrOrdered) {
1917 // UB = min(UB, GlobalUB) or
1918 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1919 // 'distribute parallel for')
1920 EmitIgnoredExpr(LoopArgs.EUB);
1921 // IV = LB
1922 EmitIgnoredExpr(LoopArgs.Init);
1923 // IV < UB
1924 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1925 } else {
1926 BoolCondVal =
1927 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1928 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1929 }
1930
1931 // If there are any cleanups between here and the loop-exit scope,
1932 // create a block to stage a loop exit along.
1933 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1934 if (LoopScope.requiresCleanups())
1935 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1936
1937 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1938 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1939 if (ExitBlock != LoopExit.getBlock()) {
1940 EmitBlock(ExitBlock);
1941 EmitBranchThroughCleanup(LoopExit);
1942 }
1943 EmitBlock(LoopBody);
1944
1945 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1946 // LB for loop condition and emitted it above).
1947 if (DynamicOrOrdered)
1948 EmitIgnoredExpr(LoopArgs.Init);
1949
1950 // Create a block for the increment.
1951 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1952 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1953
1954 // Generate !llvm.loop.parallel metadata for loads and stores for loops
1955 // with dynamic/guided scheduling and without ordered clause.
1956 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1957 LoopStack.setParallel(!IsMonotonic);
1958 else
1959 EmitOMPSimdInit(S, IsMonotonic);
1960
1961 SourceLocation Loc = S.getBeginLoc();
1962
1963 // when 'distribute' is not combined with a 'for':
1964 // while (idx <= UB) { BODY; ++idx; }
1965 // when 'distribute' is combined with a 'for'
1966 // (e.g. 'distribute parallel for')
1967 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1968 EmitOMPInnerLoop(
1969 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1970 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1971 CodeGenLoop(CGF, S, LoopExit);
1972 },
1973 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1974 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1975 });
1976
1977 EmitBlock(Continue.getBlock());
1978 BreakContinueStack.pop_back();
1979 if (!DynamicOrOrdered) {
1980 // Emit "LB = LB + Stride", "UB = UB + Stride".
1981 EmitIgnoredExpr(LoopArgs.NextLB);
1982 EmitIgnoredExpr(LoopArgs.NextUB);
1983 }
1984
1985 EmitBranch(CondBlock);
1986 LoopStack.pop();
1987 // Emit the fall-through block.
1988 EmitBlock(LoopExit.getBlock());
1989
1990 // Tell the runtime we are done.
1991 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1992 if (!DynamicOrOrdered)
1993 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1994 S.getDirectiveKind());
1995 };
1996 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1997}
1998
1999void CodeGenFunction::EmitOMPForOuterLoop(
2000 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2001 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2002 const OMPLoopArguments &LoopArgs,
2003 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2004 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2005
2006 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2007 const bool DynamicOrOrdered =
2008 Ordered || RT.isDynamic(ScheduleKind.Schedule);
2009
2010 assert((Ordered ||(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs
.Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"
) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 2013, __PRETTY_FUNCTION__))
2011 !RT.isStaticNonchunked(ScheduleKind.Schedule,(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs
.Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"
) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 2013, __PRETTY_FUNCTION__))
2012 LoopArgs.Chunk != nullptr)) &&(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs
.Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"
) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 2013, __PRETTY_FUNCTION__))
2013 "static non-chunked schedule does not need outer loop")(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs
.Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"
) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 2013, __PRETTY_FUNCTION__))
;
2014
2015 // Emit outer loop.
2016 //
2017 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2018 // When schedule(dynamic,chunk_size) is specified, the iterations are
2019 // distributed to threads in the team in chunks as the threads request them.
2020 // Each thread executes a chunk of iterations, then requests another chunk,
2021 // until no chunks remain to be distributed. Each chunk contains chunk_size
2022 // iterations, except for the last chunk to be distributed, which may have
2023 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2024 //
2025 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2026 // to threads in the team in chunks as the executing threads request them.
2027 // Each thread executes a chunk of iterations, then requests another chunk,
2028 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2029 // each chunk is proportional to the number of unassigned iterations divided
2030 // by the number of threads in the team, decreasing to 1. For a chunk_size
2031 // with value k (greater than 1), the size of each chunk is determined in the
2032 // same way, with the restriction that the chunks do not contain fewer than k
2033 // iterations (except for the last chunk to be assigned, which may have fewer
2034 // than k iterations).
2035 //
2036 // When schedule(auto) is specified, the decision regarding scheduling is
2037 // delegated to the compiler and/or runtime system. The programmer gives the
2038 // implementation the freedom to choose any possible mapping of iterations to
2039 // threads in the team.
2040 //
2041 // When schedule(runtime) is specified, the decision regarding scheduling is
2042 // deferred until run time, and the schedule and chunk size are taken from the
2043 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2044 // implementation defined
2045 //
2046 // while(__kmpc_dispatch_next(&LB, &UB)) {
2047 // idx = LB;
2048 // while (idx <= UB) { BODY; ++idx;
2049 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2050 // } // inner loop
2051 // }
2052 //
2053 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2054 // When schedule(static, chunk_size) is specified, iterations are divided into
2055 // chunks of size chunk_size, and the chunks are assigned to the threads in
2056 // the team in a round-robin fashion in the order of the thread number.
2057 //
2058 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2059 // while (idx <= UB) { BODY; ++idx; } // inner loop
2060 // LB = LB + ST;
2061 // UB = UB + ST;
2062 // }
2063 //
2064
2065 const Expr *IVExpr = S.getIterationVariable();
2066 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2067 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2068
2069 if (DynamicOrOrdered) {
2070 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2071 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
2072 llvm::Value *LBVal = DispatchBounds.first;
2073 llvm::Value *UBVal = DispatchBounds.second;
2074 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2075 LoopArgs.Chunk};
2076 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
2077 IVSigned, Ordered, DipatchRTInputValues);
2078 } else {
2079 CGOpenMPRuntime::StaticRTInput StaticInit(
2080 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2081 LoopArgs.ST, LoopArgs.Chunk);
2082 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2083 ScheduleKind, StaticInit);
2084 }
2085
2086 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
2087 const unsigned IVSize,
2088 const bool IVSigned) {
2089 if (Ordered) {
2090 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
2091 IVSigned);
2092 }
2093 };
2094
2095 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2096 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2097 OuterLoopArgs.IncExpr = S.getInc();
2098 OuterLoopArgs.Init = S.getInit();
2099 OuterLoopArgs.Cond = S.getCond();
2100 OuterLoopArgs.NextLB = S.getNextLowerBound();
2101 OuterLoopArgs.NextUB = S.getNextUpperBound();
2102 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2103 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
2104}
2105
2106static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
2107 const unsigned IVSize, const bool IVSigned) {}
2108
2109void CodeGenFunction::EmitOMPDistributeOuterLoop(
2110 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2111 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2112 const CodeGenLoopTy &CodeGenLoopContent) {
2113
2114 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2115
2116 // Emit outer loop.
2117 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2118 // dynamic
2119 //
2120
2121 const Expr *IVExpr = S.getIterationVariable();
2122 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2123 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2124
2125 CGOpenMPRuntime::StaticRTInput StaticInit(
2126 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2127 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2128 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2129
2130 // for combined 'distribute' and 'for' the increment expression of distribute
2131 // is stored in DistInc. For 'distribute' alone, it is in Inc.
2132 Expr *IncExpr;
2133 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
2134 IncExpr = S.getDistInc();
2135 else
2136 IncExpr = S.getInc();
2137
2138 // this routine is shared by 'omp distribute parallel for' and
2139 // 'omp distribute': select the right EUB expression depending on the
2140 // directive
2141 OMPLoopArguments OuterLoopArgs;
2142 OuterLoopArgs.LB = LoopArgs.LB;
2143 OuterLoopArgs.UB = LoopArgs.UB;
2144 OuterLoopArgs.ST = LoopArgs.ST;
2145 OuterLoopArgs.IL = LoopArgs.IL;
2146 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2147 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2148 ? S.getCombinedEnsureUpperBound()
2149 : S.getEnsureUpperBound();
2150 OuterLoopArgs.IncExpr = IncExpr;
2151 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2152 ? S.getCombinedInit()
2153 : S.getInit();
2154 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2155 ? S.getCombinedCond()
2156 : S.getCond();
2157 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2158 ? S.getCombinedNextLowerBound()
2159 : S.getNextLowerBound();
2160 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2161 ? S.getCombinedNextUpperBound()
2162 : S.getNextUpperBound();
2163
2164 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2165 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2166 emitEmptyOrdered);
2167}
2168
2169static std::pair<LValue, LValue>
2170emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2171 const OMPExecutableDirective &S) {
2172 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2173 LValue LB =
2174 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2175 LValue UB =
2176 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2177
2178 // When composing 'distribute' with 'for' (e.g. as in 'distribute
2179 // parallel for') we need to use the 'distribute'
2180 // chunk lower and upper bounds rather than the whole loop iteration
2181 // space. These are parameters to the outlined function for 'parallel'
2182 // and we copy the bounds of the previous schedule into the
2183 // the current ones.
2184 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2185 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2186 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2187 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2188 PrevLBVal = CGF.EmitScalarConversion(
2189 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2190 LS.getIterationVariable()->getType(),
2191 LS.getPrevLowerBoundVariable()->getExprLoc());
2192 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2193 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2194 PrevUBVal = CGF.EmitScalarConversion(
2195 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2196 LS.getIterationVariable()->getType(),
2197 LS.getPrevUpperBoundVariable()->getExprLoc());
2198
2199 CGF.EmitStoreOfScalar(PrevLBVal, LB);
2200 CGF.EmitStoreOfScalar(PrevUBVal, UB);
2201
2202 return {LB, UB};
2203}
2204
2205/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2206/// we need to use the LB and UB expressions generated by the worksharing
2207/// code generation support, whereas in non combined situations we would
2208/// just emit 0 and the LastIteration expression
2209/// This function is necessary due to the difference of the LB and UB
2210/// types for the RT emission routines for 'for_static_init' and
2211/// 'for_dispatch_init'
2212static std::pair<llvm::Value *, llvm::Value *>
2213emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2214 const OMPExecutableDirective &S,
2215 Address LB, Address UB) {
2216 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2217 const Expr *IVExpr = LS.getIterationVariable();
2218 // when implementing a dynamic schedule for a 'for' combined with a
2219 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2220 // is not normalized as each team only executes its own assigned
2221 // distribute chunk
2222 QualType IteratorTy = IVExpr->getType();
2223 llvm::Value *LBVal =
2224 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2225 llvm::Value *UBVal =
2226 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2227 return {LBVal, UBVal};
2228}
2229
2230static void emitDistributeParallelForDistributeInnerBoundParams(
2231 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2232 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2233 const auto &Dir = cast<OMPLoopDirective>(S);
2234 LValue LB =
2235 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2236 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2237 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2238 CapturedVars.push_back(LBCast);
2239 LValue UB =
2240 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2241
2242 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2243 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2244 CapturedVars.push_back(UBCast);
2245}
2246
2247static void
2248emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2249 const OMPLoopDirective &S,
2250 CodeGenFunction::JumpDest LoopExit) {
2251 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2252 PrePostActionTy &Action) {
2253 Action.Enter(CGF);
2254 bool HasCancel = false;
2255 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2256 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2257 HasCancel = D->hasCancel();
2258 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2259 HasCancel = D->hasCancel();
2260 else if (const auto *D =
2261 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2262 HasCancel = D->hasCancel();
2263 }
2264 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2265 HasCancel);
2266 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2267 emitDistributeParallelForInnerBounds,
2268 emitDistributeParallelForDispatchBounds);
2269 };
2270
2271 emitCommonOMPParallelDirective(
2272 CGF, S,
2273 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2274 CGInlinedWorksharingLoop,
2275 emitDistributeParallelForDistributeInnerBoundParams);
2276}
2277
2278void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2279 const OMPDistributeParallelForDirective &S) {
2280 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2281 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2282 S.getDistInc());
2283 };
2284 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2285 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2286}
2287
2288void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2289 const OMPDistributeParallelForSimdDirective &S) {
2290 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2291 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2292 S.getDistInc());
2293 };
2294 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2295 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2296}
2297
2298void CodeGenFunction::EmitOMPDistributeSimdDirective(
2299 const OMPDistributeSimdDirective &S) {
2300 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2301 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2302 };
2303 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2304 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2305}
2306
2307void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2308 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2309 // Emit SPMD target parallel for region as a standalone region.
2310 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2311 emitOMPSimdRegion(CGF, S, Action);
2312 };
2313 llvm::Function *Fn;
2314 llvm::Constant *Addr;
2315 // Emit target region as a standalone region.
2316 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2317 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2318 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 2318, __PRETTY_FUNCTION__))
;
2319}
2320
2321void CodeGenFunction::EmitOMPTargetSimdDirective(
2322 const OMPTargetSimdDirective &S) {
2323 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2324 emitOMPSimdRegion(CGF, S, Action);
2325 };
2326 emitCommonOMPTargetDirective(*this, S, CodeGen);
2327}
2328
2329namespace {
2330 struct ScheduleKindModifiersTy {
2331 OpenMPScheduleClauseKind Kind;
2332 OpenMPScheduleClauseModifier M1;
2333 OpenMPScheduleClauseModifier M2;
2334 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2335 OpenMPScheduleClauseModifier M1,
2336 OpenMPScheduleClauseModifier M2)
2337 : Kind(Kind), M1(M1), M2(M2) {}
2338 };
2339} // namespace
2340
2341bool CodeGenFunction::EmitOMPWorksharingLoop(
2342 const OMPLoopDirective &S, Expr *EUB,
2343 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2344 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2345 // Emit the loop iteration variable.
2346 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2347 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2348 EmitVarDecl(*IVDecl);
2349
2350 // Emit the iterations count variable.
2351 // If it is not a variable, Sema decided to calculate iterations count on each
2352 // iteration (e.g., it is foldable into a constant).
2353 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2354 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2355 // Emit calculation of the iterations count.
2356 EmitIgnoredExpr(S.getCalcLastIteration());
2357 }
2358
2359 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2360
2361 bool HasLastprivateClause;
2362 // Check pre-condition.
2363 {
2364 OMPLoopScope PreInitScope(*this, S);
2365 // Skip the entire loop if we don't meet the precondition.
2366 // If the condition constant folds and can be elided, avoid emitting the
2367 // whole loop.
2368 bool CondConstant;
2369 llvm::BasicBlock *ContBlock = nullptr;
2370 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2371 if (!CondConstant)
2372 return false;
2373 } else {
2374 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2375 ContBlock = createBasicBlock("omp.precond.end");
2376 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2377 getProfileCount(&S));
2378 EmitBlock(ThenBlock);
2379 incrementProfileCounter(&S);
2380 }
2381
2382 RunCleanupsScope DoacrossCleanupScope(*this);
2383 bool Ordered = false;
2384 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2385 if (OrderedClause->getNumForLoops())
2386 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2387 else
2388 Ordered = true;
2389 }
2390
2391 llvm::DenseSet<const Expr *> EmittedFinals;
2392 emitAlignedClause(*this, S);
2393 bool HasLinears = EmitOMPLinearClauseInit(S);
2394 // Emit helper vars inits.
2395
2396 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2397 LValue LB = Bounds.first;
2398 LValue UB = Bounds.second;
2399 LValue ST =
2400 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2401 LValue IL =
2402 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2403
2404 // Emit 'then' code.
2405 {
2406 OMPPrivateScope LoopScope(*this);
2407 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2408 // Emit implicit barrier to synchronize threads and avoid data races on
2409 // initialization of firstprivate variables and post-update of
2410 // lastprivate variables.
2411 CGM.getOpenMPRuntime().emitBarrierCall(
2412 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2413 /*ForceSimpleCall=*/true);
2414 }
2415 EmitOMPPrivateClause(S, LoopScope);
2416 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2417 EmitOMPReductionClauseInit(S, LoopScope);
2418 EmitOMPPrivateLoopCounters(S, LoopScope);
2419 EmitOMPLinearClause(S, LoopScope);
2420 (void)LoopScope.Privatize();
2421 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2422 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2423
2424 // Detect the loop schedule kind and chunk.
2425 const Expr *ChunkExpr = nullptr;
2426 OpenMPScheduleTy ScheduleKind;
2427 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2428 ScheduleKind.Schedule = C->getScheduleKind();
2429 ScheduleKind.M1 = C->getFirstScheduleModifier();
2430 ScheduleKind.M2 = C->getSecondScheduleModifier();
2431 ChunkExpr = C->getChunkSize();
2432 } else {
2433 // Default behaviour for schedule clause.
2434 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2435 *this, S, ScheduleKind.Schedule, ChunkExpr);
2436 }
2437 bool HasChunkSizeOne = false;
2438 llvm::Value *Chunk = nullptr;
2439 if (ChunkExpr) {
2440 Chunk = EmitScalarExpr(ChunkExpr);
2441 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2442 S.getIterationVariable()->getType(),
2443 S.getBeginLoc());
2444 Expr::EvalResult Result;
2445 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2446 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2447 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2448 }
2449 }
2450 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2451 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2452 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2453 // If the static schedule kind is specified or if the ordered clause is
2454 // specified, and if no monotonic modifier is specified, the effect will
2455 // be as if the monotonic modifier was specified.
2456 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2457 /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2458 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
2459 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2460 /* Chunked */ Chunk != nullptr) ||
2461 StaticChunkedOne) &&
2462 !Ordered) {
2463 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2464 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2465 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2466 // When no chunk_size is specified, the iteration space is divided into
2467 // chunks that are approximately equal in size, and at most one chunk is
2468 // distributed to each thread. Note that the size of the chunks is
2469 // unspecified in this case.
2470 CGOpenMPRuntime::StaticRTInput StaticInit(
2471 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2472 UB.getAddress(), ST.getAddress(),
2473 StaticChunkedOne ? Chunk : nullptr);
2474 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2475 ScheduleKind, StaticInit);
2476 JumpDest LoopExit =
2477 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2478 // UB = min(UB, GlobalUB);
2479 if (!StaticChunkedOne)
2480 EmitIgnoredExpr(S.getEnsureUpperBound());
2481 // IV = LB;
2482 EmitIgnoredExpr(S.getInit());
2483 // For unchunked static schedule generate:
2484 //
2485 // while (idx <= UB) {
2486 // BODY;
2487 // ++idx;
2488 // }
2489 //
2490 // For static schedule with chunk one:
2491 //
2492 // while (IV <= PrevUB) {
2493 // BODY;
2494 // IV += ST;
2495 // }
2496 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2497 StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2498 StaticChunkedOne ? S.getDistInc() : S.getInc(),
2499 [&S, LoopExit](CodeGenFunction &CGF) {
2500 CGF.EmitOMPLoopBody(S, LoopExit);
2501 CGF.EmitStopPoint(&S);
2502 },
2503 [](CodeGenFunction &) {});
2504 EmitBlock(LoopExit.getBlock());
2505 // Tell the runtime we are done.
2506 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2507 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2508 S.getDirectiveKind());
2509 };
2510 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2511 } else {
2512 const bool IsMonotonic =
2513 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2514 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2515 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2516 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2517 // Emit the outer loop, which requests its work chunk [LB..UB] from
2518 // runtime and runs the inner loop to process it.
2519 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2520 ST.getAddress(), IL.getAddress(),
2521 Chunk, EUB);
2522 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2523 LoopArguments, CGDispatchBounds);
2524 }
2525 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2526 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2527 return CGF.Builder.CreateIsNotNull(
2528 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2529 });
2530 }
2531 EmitOMPReductionClauseFinal(
2532 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2533 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2534 : /*Parallel only*/ OMPD_parallel);
2535 // Emit post-update of the reduction variables if IsLastIter != 0.
2536 emitPostUpdateForReductionClause(
2537 *this, S, [IL, &S](CodeGenFunction &CGF) {
2538 return CGF.Builder.CreateIsNotNull(
2539 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2540 });
2541 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2542 if (HasLastprivateClause)
2543 EmitOMPLastprivateClauseFinal(
2544 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2545 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2546 }
2547 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2548 return CGF.Builder.CreateIsNotNull(
2549 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2550 });
2551 DoacrossCleanupScope.ForceCleanup();
2552 // We're now done with the loop, so jump to the continuation block.
2553 if (ContBlock) {
2554 EmitBranch(ContBlock);
2555 EmitBlock(ContBlock, /*IsFinished=*/true);
2556 }
2557 }
2558 return HasLastprivateClause;
2559}
2560
2561/// The following two functions generate expressions for the loop lower
2562/// and upper bounds in case of static and dynamic (dispatch) schedule
2563/// of the associated 'for' or 'distribute' loop.
2564static std::pair<LValue, LValue>
2565emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2566 const auto &LS = cast<OMPLoopDirective>(S);
2567 LValue LB =
2568 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2569 LValue UB =
2570 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2571 return {LB, UB};
2572}
2573
2574/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2575/// consider the lower and upper bound expressions generated by the
2576/// worksharing loop support, but we use 0 and the iteration space size as
2577/// constants
2578static std::pair<llvm::Value *, llvm::Value *>
2579emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2580 Address LB, Address UB) {
2581 const auto &LS = cast<OMPLoopDirective>(S);
2582 const Expr *IVExpr = LS.getIterationVariable();
2583 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2584 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2585 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2586 return {LBVal, UBVal};
2587}
2588
2589void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2590 bool HasLastprivates = false;
2591 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2592 PrePostActionTy &) {
2593 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2594 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2595 emitForLoopBounds,
2596 emitDispatchForLoopBounds);
2597 };
2598 {
2599 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2600 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2601 S.hasCancel());
2602 }
2603
2604 // Emit an implicit barrier at the end.
2605 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2606 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2607}
2608
2609void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2610 bool HasLastprivates = false;
2611 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2612 PrePostActionTy &) {
2613 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2614 emitForLoopBounds,
2615 emitDispatchForLoopBounds);
2616 };
2617 {
2618 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2619 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2620 }
2621
2622 // Emit an implicit barrier at the end.
2623 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2624 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2625}
2626
2627static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2628 const Twine &Name,
2629 llvm::Value *Init = nullptr) {
2630 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2631 if (Init)
2632 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2633 return LVal;
2634}
2635
2636void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2637 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
2638 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2639 bool HasLastprivates = false;
2640 auto &&CodeGen = [&S, CapturedStmt, CS,
2641 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2642 ASTContext &C = CGF.getContext();
2643 QualType KmpInt32Ty =
2644 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2645 // Emit helper vars inits.
2646 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2647 CGF.Builder.getInt32(0));
2648 llvm::ConstantInt *GlobalUBVal = CS != nullptr
2649 ? CGF.Builder.getInt32(CS->size() - 1)
2650 : CGF.Builder.getInt32(0);
2651 LValue UB =
2652 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2653 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2654 CGF.Builder.getInt32(1));
2655 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2656 CGF.Builder.getInt32(0));
2657 // Loop counter.
2658 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2659 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2660 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2661 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2662 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2663 // Generate condition for loop.
2664 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2665 OK_Ordinary, S.getBeginLoc(), FPOptions());
2666 // Increment for loop counter.
2667 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2668 S.getBeginLoc(), true);
2669 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2670 // Iterate through all sections and emit a switch construct:
2671 // switch (IV) {
2672 // case 0:
2673 // <SectionStmt[0]>;
2674 // break;
2675 // ...
2676 // case <NumSection> - 1:
2677 // <SectionStmt[<NumSection> - 1]>;
2678 // break;
2679 // }
2680 // .omp.sections.exit:
2681 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2682 llvm::SwitchInst *SwitchStmt =
2683 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2684 ExitBB, CS == nullptr ? 1 : CS->size());
2685 if (CS) {
2686 unsigned CaseNumber = 0;
2687 for (const Stmt *SubStmt : CS->children()) {
2688 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2689 CGF.EmitBlock(CaseBB);
2690 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2691 CGF.EmitStmt(SubStmt);
2692 CGF.EmitBranch(ExitBB);
2693 ++CaseNumber;
2694 }
2695 } else {
2696 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2697 CGF.EmitBlock(CaseBB);
2698 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2699 CGF.EmitStmt(CapturedStmt);
2700 CGF.EmitBranch(ExitBB);
2701 }
2702 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2703 };
2704
2705 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2706 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2707 // Emit implicit barrier to synchronize threads and avoid data races on
2708 // initialization of firstprivate variables and post-update of lastprivate
2709 // variables.
2710 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2711 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2712 /*ForceSimpleCall=*/true);
2713 }
2714 CGF.EmitOMPPrivateClause(S, LoopScope);
2715 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2716 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2717 (void)LoopScope.Privatize();
2718 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2719 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2720
2721 // Emit static non-chunked loop.
2722 OpenMPScheduleTy ScheduleKind;
2723 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2724 CGOpenMPRuntime::StaticRTInput StaticInit(
2725 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2726 LB.getAddress(), UB.getAddress(), ST.getAddress());
2727 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2728 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2729 // UB = min(UB, GlobalUB);
2730 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2731 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2732 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2733 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2734 // IV = LB;
2735 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2736 // while (idx <= UB) { BODY; ++idx; }
2737 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2738 [](CodeGenFunction &) {});
2739 // Tell the runtime we are done.
2740 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2741 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2742 S.getDirectiveKind());
2743 };
2744 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2745 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2746 // Emit post-update of the reduction variables if IsLastIter != 0.
2747 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2748 return CGF.Builder.CreateIsNotNull(
2749 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2750 });
2751
2752 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2753 if (HasLastprivates)
2754 CGF.EmitOMPLastprivateClauseFinal(
2755 S, /*NoFinals=*/false,
2756 CGF.Builder.CreateIsNotNull(
2757 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2758 };
2759
2760 bool HasCancel = false;
2761 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2762 HasCancel = OSD->hasCancel();
2763 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2764 HasCancel = OPSD->hasCancel();
2765 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2766 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2767 HasCancel);
2768 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2769 // clause. Otherwise the barrier will be generated by the codegen for the
2770 // directive.
2771 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2772 // Emit implicit barrier to synchronize threads and avoid data races on
2773 // initialization of firstprivate variables.
2774 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2775 OMPD_unknown);
2776 }
2777}
2778
2779void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2780 {
2781 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2782 EmitSections(S);
2783 }
2784 // Emit an implicit barrier at the end.
2785 if (!S.getSingleClause<OMPNowaitClause>()) {
2786 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2787 OMPD_sections);
2788 }
2789}
2790
2791void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2792 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2793 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2794 };
2795 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2796 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2797 S.hasCancel());
2798}
2799
2800void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2801 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2802 llvm::SmallVector<const Expr *, 8> DestExprs;
2803 llvm::SmallVector<const Expr *, 8> SrcExprs;
2804 llvm::SmallVector<const Expr *, 8> AssignmentOps;
2805 // Check if there are any 'copyprivate' clauses associated with this
2806 // 'single' construct.
2807 // Build a list of copyprivate variables along with helper expressions
2808 // (<source>, <destination>, <destination>=<source> expressions)
2809 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2810 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2811 DestExprs.append(C->destination_exprs().begin(),
2812 C->destination_exprs().end());
2813 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2814 AssignmentOps.append(C->assignment_ops().begin(),
2815 C->assignment_ops().end());
2816 }
2817 // Emit code for 'single' region along with 'copyprivate' clauses
2818 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2819 Action.Enter(CGF);
2820 OMPPrivateScope SingleScope(CGF);
2821 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2822 CGF.EmitOMPPrivateClause(S, SingleScope);
2823 (void)SingleScope.Privatize();
2824 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2825 };
2826 {
2827 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2828 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2829 CopyprivateVars, DestExprs,
2830 SrcExprs, AssignmentOps);
2831 }
2832 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2833 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2834 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2835 CGM.getOpenMPRuntime().emitBarrierCall(
2836 *this, S.getBeginLoc(),
2837 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2838 }
2839}
2840
2841void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
2842 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2843 Action.Enter(CGF);
2844 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2845 };
2846 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2847 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2848}
2849
2850void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
2851 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2852 Action.Enter(CGF);
2853 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2854 };
2855 const Expr *Hint = nullptr;
2856 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2857 Hint = HintClause->getHint();
2858 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2859 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2860 S.getDirectiveName().getAsString(),
2861 CodeGen, S.getBeginLoc(), Hint);
2862}
2863
2864void CodeGenFunction::EmitOMPParallelForDirective(
2865 const OMPParallelForDirective &S) {
2866 // Emit directive as a combined directive that consists of two implicit
2867 // directives: 'parallel' with 'for' directive.
2868 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2869 Action.Enter(CGF);
2870 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2871 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2872 emitDispatchForLoopBounds);
2873 };
2874 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2875 emitEmptyBoundParameters);
2876}
2877
2878void CodeGenFunction::EmitOMPParallelForSimdDirective(
2879 const OMPParallelForSimdDirective &S) {
2880 // Emit directive as a combined directive that consists of two implicit
2881 // directives: 'parallel' with 'for' directive.
2882 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2883 Action.Enter(CGF);
2884 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2885 emitDispatchForLoopBounds);
2886 };
2887 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2888 emitEmptyBoundParameters);
2889}
2890
2891void CodeGenFunction::EmitOMPParallelSectionsDirective(
2892 const OMPParallelSectionsDirective &S) {
2893 // Emit directive as a combined directive that consists of two implicit
2894 // directives: 'parallel' with 'sections' directive.
2895 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2896 Action.Enter(CGF);
2897 CGF.EmitSections(S);
2898 };
2899 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2900 emitEmptyBoundParameters);
2901}
2902
2903void CodeGenFunction::EmitOMPTaskBasedDirective(
2904 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2905 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2906 OMPTaskDataTy &Data) {
2907 // Emit outlined function for task construct.
2908 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2909 auto I = CS->getCapturedDecl()->param_begin();
2910 auto PartId = std::next(I);
2911 auto TaskT = std::next(I, 4);
2912 // Check if the task is final
2913 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2914 // If the condition constant folds and can be elided, try to avoid emitting
2915 // the condition and the dead arm of the if/else.
2916 const Expr *Cond = Clause->getCondition();
2917 bool CondConstant;
2918 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2919 Data.Final.setInt(CondConstant);
2920 else
2921 Data.Final.setPointer(EvaluateExprAsBool(Cond));
2922 } else {
2923 // By default the task is not final.
2924 Data.Final.setInt(/*IntVal=*/false);
2925 }
2926 // Check if the task has 'priority' clause.
2927 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2928 const Expr *Prio = Clause->getPriority();
2929 Data.Priority.setInt(/*IntVal=*/true);
2930 Data.Priority.setPointer(EmitScalarConversion(
2931 EmitScalarExpr(Prio), Prio->getType(),
2932 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2933 Prio->getExprLoc()));
2934 }
2935 // The first function argument for tasks is a thread id, the second one is a
2936 // part id (0 for tied tasks, >=0 for untied task).
2937 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2938 // Get list of private variables.
2939 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2940 auto IRef = C->varlist_begin();
2941 for (const Expr *IInit : C->private_copies()) {
2942 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2943 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2944 Data.PrivateVars.push_back(*IRef);
2945 Data.PrivateCopies.push_back(IInit);
2946 }
2947 ++IRef;
2948 }
2949 }
2950 EmittedAsPrivate.clear();
2951 // Get list of firstprivate variables.
2952 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2953 auto IRef = C->varlist_begin();
2954 auto IElemInitRef = C->inits().begin();
2955 for (const Expr *IInit : C->private_copies()) {
2956 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2957 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2958 Data.FirstprivateVars.push_back(*IRef);
2959 Data.FirstprivateCopies.push_back(IInit);
2960 Data.FirstprivateInits.push_back(*IElemInitRef);
2961 }
2962 ++IRef;
2963 ++IElemInitRef;
2964 }
2965 }
2966 // Get list of lastprivate variables (for taskloops).
2967 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2968 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2969 auto IRef = C->varlist_begin();
2970 auto ID = C->destination_exprs().begin();
2971 for (const Expr *IInit : C->private_copies()) {
2972 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2974 Data.LastprivateVars.push_back(*IRef);
2975 Data.LastprivateCopies.push_back(IInit);
2976 }
2977 LastprivateDstsOrigs.insert(
2978 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2979 cast<DeclRefExpr>(*IRef)});
2980 ++IRef;
2981 ++ID;
2982 }
2983 }
2984 SmallVector<const Expr *, 4> LHSs;
2985 SmallVector<const Expr *, 4> RHSs;
2986 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2987 auto IPriv = C->privates().begin();
2988 auto IRed = C->reduction_ops().begin();
2989 auto ILHS = C->lhs_exprs().begin();
2990 auto IRHS = C->rhs_exprs().begin();
2991 for (const Expr *Ref : C->varlists()) {
2992 Data.ReductionVars.emplace_back(Ref);
2993 Data.ReductionCopies.emplace_back(*IPriv);
2994 Data.ReductionOps.emplace_back(*IRed);
2995 LHSs.emplace_back(*ILHS);
2996 RHSs.emplace_back(*IRHS);
2997 std::advance(IPriv, 1);
2998 std::advance(IRed, 1);
2999 std::advance(ILHS, 1);
3000 std::advance(IRHS, 1);
3001 }
3002 }
3003 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
3004 *this, S.getBeginLoc(), LHSs, RHSs, Data);
3005 // Build list of dependences.
3006 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3007 for (const Expr *IRef : C->varlists())
3008 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3009 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
3010 CapturedRegion](CodeGenFunction &CGF,
3011 PrePostActionTy &Action) {
3012 // Set proper addresses for generated private copies.
3013 OMPPrivateScope Scope(CGF);
3014 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
3015 !Data.LastprivateVars.empty()) {
3016 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3017 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3018 enum { PrivatesParam = 2, CopyFnParam = 3 };
3019 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3020 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3021 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3022 CS->getCapturedDecl()->getParam(PrivatesParam)));
3023 // Map privates.
3024 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3025 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3026 CallArgs.push_back(PrivatesPtr);
3027 for (const Expr *E : Data.PrivateVars) {
3028 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3029 Address PrivatePtr = CGF.CreateMemTemp(
3030 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
3031 PrivatePtrs.emplace_back(VD, PrivatePtr);
3032 CallArgs.push_back(PrivatePtr.getPointer());
3033 }
3034 for (const Expr *E : Data.FirstprivateVars) {
3035 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3036 Address PrivatePtr =
3037 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3038 ".firstpriv.ptr.addr");
3039 PrivatePtrs.emplace_back(VD, PrivatePtr);
3040 CallArgs.push_back(PrivatePtr.getPointer());
3041 }
3042 for (const Expr *E : Data.LastprivateVars) {
3043 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3044 Address PrivatePtr =
3045 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3046 ".lastpriv.ptr.addr");
3047 PrivatePtrs.emplace_back(VD, PrivatePtr);
3048 CallArgs.push_back(PrivatePtr.getPointer());
3049 }
3050 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3051 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3052 for (const auto &Pair : LastprivateDstsOrigs) {
3053 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
3054 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
3055 /*RefersToEnclosingVariableOrCapture=*/
3056 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
3057 Pair.second->getType(), VK_LValue,
3058 Pair.second->getExprLoc());
3059 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
3060 return CGF.EmitLValue(&DRE).getAddress();
3061 });
3062 }
3063 for (const auto &Pair : PrivatePtrs) {
3064 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3065 CGF.getContext().getDeclAlign(Pair.first));
3066 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3067 }
3068 }
3069 if (Data.Reductions) {
3070 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
3071 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
3072 Data.ReductionOps);
3073 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
3074 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
3075 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
3076 RedCG.emitSharedLValue(CGF, Cnt);
3077 RedCG.emitAggregateType(CGF, Cnt);
3078 // FIXME: This must removed once the runtime library is fixed.
3079 // Emit required threadprivate variables for
3080 // initializer/combiner/finalizer.
3081 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3082 RedCG, Cnt);
3083 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3084 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3085 Replacement =
3086 Address(CGF.EmitScalarConversion(
3087 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3088 CGF.getContext().getPointerType(
3089 Data.ReductionCopies[Cnt]->getType()),
3090 Data.ReductionCopies[Cnt]->getExprLoc()),
3091 Replacement.getAlignment());
3092 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3093 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
3094 [Replacement]() { return Replacement; });
3095 }
3096 }
3097 // Privatize all private variables except for in_reduction items.
3098 (void)Scope.Privatize();
3099 SmallVector<const Expr *, 4> InRedVars;
3100 SmallVector<const Expr *, 4> InRedPrivs;
3101 SmallVector<const Expr *, 4> InRedOps;
3102 SmallVector<const Expr *, 4> TaskgroupDescriptors;
3103 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
3104 auto IPriv = C->privates().begin();
3105 auto IRed = C->reduction_ops().begin();
3106 auto ITD = C->taskgroup_descriptors().begin();
3107 for (const Expr *Ref : C->varlists()) {
3108 InRedVars.emplace_back(Ref);
3109 InRedPrivs.emplace_back(*IPriv);
3110 InRedOps.emplace_back(*IRed);
3111 TaskgroupDescriptors.emplace_back(*ITD);
3112 std::advance(IPriv, 1);
3113 std::advance(IRed, 1);
3114 std::advance(ITD, 1);
3115 }
3116 }
3117 // Privatize in_reduction items here, because taskgroup descriptors must be
3118 // privatized earlier.
3119 OMPPrivateScope InRedScope(CGF);
3120 if (!InRedVars.empty()) {
3121 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3122 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3123 RedCG.emitSharedLValue(CGF, Cnt);
3124 RedCG.emitAggregateType(CGF, Cnt);
3125 // The taskgroup descriptor variable is always implicit firstprivate and
3126 // privatized already during processing of the firstprivates.
3127 // FIXME: This must removed once the runtime library is fixed.
3128 // Emit required threadprivate variables for
3129 // initializer/combiner/finalizer.
3130 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3131 RedCG, Cnt);
3132 llvm::Value *ReductionsPtr =
3133 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3134 TaskgroupDescriptors[Cnt]->getExprLoc());
3135 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3136 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3137 Replacement = Address(
3138 CGF.EmitScalarConversion(
3139 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3140 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3141 InRedPrivs[Cnt]->getExprLoc()),
3142 Replacement.getAlignment());
3143 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3144 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3145 [Replacement]() { return Replacement; });
3146 }
3147 }
3148 (void)InRedScope.Privatize();
3149
3150 Action.Enter(CGF);
3151 BodyGen(CGF);
3152 };
3153 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3154 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3155 Data.NumberOfParts);
3156 OMPLexicalScope Scope(*this, S, llvm::None,
3157 !isOpenMPParallelDirective(S.getDirectiveKind()));
3158 TaskGen(*this, OutlinedFn, Data);
3159}
3160
3161static ImplicitParamDecl *
3162createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
3163 QualType Ty, CapturedDecl *CD,
3164 SourceLocation Loc) {
3165 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3166 ImplicitParamDecl::Other);
3167 auto *OrigRef = DeclRefExpr::Create(
3168 C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3169 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3170 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3171 ImplicitParamDecl::Other);
3172 auto *PrivateRef = DeclRefExpr::Create(
3173 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3174 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3175 QualType ElemType = C.getBaseElementType(Ty);
3176 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3177 ImplicitParamDecl::Other);
3178 auto *InitRef = DeclRefExpr::Create(
3179 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3180 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3181 PrivateVD->setInitStyle(VarDecl::CInit);
3182 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3183 InitRef, /*BasePath=*/nullptr,
3184 VK_RValue));
3185 Data.FirstprivateVars.emplace_back(OrigRef);
3186 Data.FirstprivateCopies.emplace_back(PrivateRef);
3187 Data.FirstprivateInits.emplace_back(InitRef);
3188 return OrigVD;
3189}
3190
3191void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3192 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3193 OMPTargetDataInfo &InputInfo) {
3194 // Emit outlined function for task construct.
3195 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3196 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3197 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3198 auto I = CS->getCapturedDecl()->param_begin();
3199 auto PartId = std::next(I);
3200 auto TaskT = std::next(I, 4);
3201 OMPTaskDataTy Data;
3202 // The task is not final.
3203 Data.Final.setInt(/*IntVal=*/false);
3204 // Get list of firstprivate variables.
3205 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3206 auto IRef = C->varlist_begin();
3207 auto IElemInitRef = C->inits().begin();
3208 for (auto *IInit : C->private_copies()) {
3209 Data.FirstprivateVars.push_back(*IRef);
3210 Data.FirstprivateCopies.push_back(IInit);
3211 Data.FirstprivateInits.push_back(*IElemInitRef);
3212 ++IRef;
3213 ++IElemInitRef;
3214 }
3215 }
3216 OMPPrivateScope TargetScope(*this);
3217 VarDecl *BPVD = nullptr;
3218 VarDecl *PVD = nullptr;
3219 VarDecl *SVD = nullptr;
3220 if (InputInfo.NumberOfTargetItems > 0) {
3221 auto *CD = CapturedDecl::Create(
3222 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3223 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3224 QualType BaseAndPointersType = getContext().getConstantArrayType(
3225 getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal,
3226 /*IndexTypeQuals=*/0);
3227 BPVD = createImplicitFirstprivateForType(
3228 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3229 PVD = createImplicitFirstprivateForType(
3230 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3231 QualType SizesType = getContext().getConstantArrayType(
3232 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
3233 ArrSize, nullptr, ArrayType::Normal,
3234 /*IndexTypeQuals=*/0);
3235 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3236 S.getBeginLoc());
3237 TargetScope.addPrivate(
3238 BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3239 TargetScope.addPrivate(PVD,
3240 [&InputInfo]() { return InputInfo.PointersArray; });
3241 TargetScope.addPrivate(SVD,
3242 [&InputInfo]() { return InputInfo.SizesArray; });
3243 }
3244 (void)TargetScope.Privatize();
3245 // Build list of dependences.
3246 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3247 for (const Expr *IRef : C->varlists())
3248 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3249 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3250 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3251 // Set proper addresses for generated private copies.
3252 OMPPrivateScope Scope(CGF);
3253 if (!Data.FirstprivateVars.empty()) {
3254 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3255 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3256 enum { PrivatesParam = 2, CopyFnParam = 3 };
3257 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3258 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3259 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3260 CS->getCapturedDecl()->getParam(PrivatesParam)));
3261 // Map privates.
3262 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3263 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3264 CallArgs.push_back(PrivatesPtr);
3265 for (const Expr *E : Data.FirstprivateVars) {
3266 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3267 Address PrivatePtr =
3268 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3269 ".firstpriv.ptr.addr");
3270 PrivatePtrs.emplace_back(VD, PrivatePtr);
3271 CallArgs.push_back(PrivatePtr.getPointer());
3272 }
3273 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3274 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3275 for (const auto &Pair : PrivatePtrs) {
3276 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3277 CGF.getContext().getDeclAlign(Pair.first));
3278 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3279 }
3280 }
3281 // Privatize all private variables except for in_reduction items.
3282 (void)Scope.Privatize();
3283 if (InputInfo.NumberOfTargetItems > 0) {
3284 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3285 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3286 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3287 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3288 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3289 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3290 }
3291
3292 Action.Enter(CGF);
3293 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3294 BodyGen(CGF);
3295 };
3296 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3297 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3298 Data.NumberOfParts);
3299 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3300 IntegerLiteral IfCond(getContext(), TrueOrFalse,
3301 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3302 SourceLocation());
3303
3304 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3305 SharedsTy, CapturedStruct, &IfCond, Data);
3306}
3307
3308void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3309 // Emit outlined function for task construct.
3310 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3311 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3312 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3313 const Expr *IfCond = nullptr;
3314 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3315 if (C->getNameModifier() == OMPD_unknown ||
3316 C->getNameModifier() == OMPD_task) {
3317 IfCond = C->getCondition();
3318 break;
3319 }
3320 }
3321
3322 OMPTaskDataTy Data;
3323 // Check if we should emit tied or untied task.
3324 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3325 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3326 CGF.EmitStmt(CS->getCapturedStmt());
3327 };
3328 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3329 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3330 const OMPTaskDataTy &Data) {
3331 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3332 SharedsTy, CapturedStruct, IfCond,
3333 Data);
3334 };
3335 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3336}
3337
3338void CodeGenFunction::EmitOMPTaskyieldDirective(
3339 const OMPTaskyieldDirective &S) {
3340 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3341}
3342
3343void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
3344 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3345}
3346
3347void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
3348 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3349}
3350
3351void CodeGenFunction::EmitOMPTaskgroupDirective(
3352 const OMPTaskgroupDirective &S) {
3353 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3354 Action.Enter(CGF);
3355 if (const Expr *E = S.getReductionRef()) {
3356 SmallVector<const Expr *, 4> LHSs;
3357 SmallVector<const Expr *, 4> RHSs;
3358 OMPTaskDataTy Data;
3359 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3360 auto IPriv = C->privates().begin();
3361 auto IRed = C->reduction_ops().begin();
3362 auto ILHS = C->lhs_exprs().begin();
3363 auto IRHS = C->rhs_exprs().begin();
3364 for (const Expr *Ref : C->varlists()) {
3365 Data.ReductionVars.emplace_back(Ref);
3366 Data.ReductionCopies.emplace_back(*IPriv);
3367 Data.ReductionOps.emplace_back(*IRed);
3368 LHSs.emplace_back(*ILHS);
3369 RHSs.emplace_back(*IRHS);
3370 std::advance(IPriv, 1);
3371 std::advance(IRed, 1);
3372 std::advance(ILHS, 1);
3373 std::advance(IRHS, 1);
3374 }
3375 }
3376 llvm::Value *ReductionDesc =
3377 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3378 LHSs, RHSs, Data);
3379 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3380 CGF.EmitVarDecl(*VD);
3381 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3382 /*Volatile=*/false, E->getType());
3383 }
3384 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3385 };
3386 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3387 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3388}
3389
3390void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
3391 CGM.getOpenMPRuntime().emitFlush(
3392 *this,
3393 [&S]() -> ArrayRef<const Expr *> {
3394 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3395 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3396 FlushClause->varlist_end());
3397 return llvm::None;
3398 }(),
3399 S.getBeginLoc());
3400}
3401
3402void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3403 const CodeGenLoopTy &CodeGenLoop,
3404 Expr *IncExpr) {
3405 // Emit the loop iteration variable.
3406 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3407 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3408 EmitVarDecl(*IVDecl);
3409
3410 // Emit the iterations count variable.
3411 // If it is not a variable, Sema decided to calculate iterations count on each
3412 // iteration (e.g., it is foldable into a constant).
3413 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3414 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3415 // Emit calculation of the iterations count.
3416 EmitIgnoredExpr(S.getCalcLastIteration());
3417 }
3418
3419 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3420
3421 bool HasLastprivateClause = false;
3422 // Check pre-condition.
3423 {
3424 OMPLoopScope PreInitScope(*this, S);
3425 // Skip the entire loop if we don't meet the precondition.
3426 // If the condition constant folds and can be elided, avoid emitting the
3427 // whole loop.
3428 bool CondConstant;
3429 llvm::BasicBlock *ContBlock = nullptr;
3430 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3431 if (!CondConstant)
3432 return;
3433 } else {
3434 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3435 ContBlock = createBasicBlock("omp.precond.end");
3436 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3437 getProfileCount(&S));
3438 EmitBlock(ThenBlock);
3439 incrementProfileCounter(&S);
3440 }
3441
3442 emitAlignedClause(*this, S);
3443 // Emit 'then' code.
3444 {
3445 // Emit helper vars inits.
3446
3447 LValue LB = EmitOMPHelperVar(
3448 *this, cast<DeclRefExpr>(
3449 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3450 ? S.getCombinedLowerBoundVariable()
3451 : S.getLowerBoundVariable())));
3452 LValue UB = EmitOMPHelperVar(
3453 *this, cast<DeclRefExpr>(
3454 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3455 ? S.getCombinedUpperBoundVariable()
3456 : S.getUpperBoundVariable())));
3457 LValue ST =
3458 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3459 LValue IL =
3460 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3461
3462 OMPPrivateScope LoopScope(*this);
3463 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3464 // Emit implicit barrier to synchronize threads and avoid data races
3465 // on initialization of firstprivate variables and post-update of
3466 // lastprivate variables.
3467 CGM.getOpenMPRuntime().emitBarrierCall(
3468 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3469 /*ForceSimpleCall=*/true);
3470 }
3471 EmitOMPPrivateClause(S, LoopScope);
3472 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3473 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3474 !isOpenMPTeamsDirective(S.getDirectiveKind()))
3475 EmitOMPReductionClauseInit(S, LoopScope);
3476 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3477 EmitOMPPrivateLoopCounters(S, LoopScope);
3478 (void)LoopScope.Privatize();
3479 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3480 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3481
3482 // Detect the distribute schedule kind and chunk.
3483 llvm::Value *Chunk = nullptr;
3484 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3485 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3486 ScheduleKind = C->getDistScheduleKind();
3487 if (const Expr *Ch = C->getChunkSize()) {
3488 Chunk = EmitScalarExpr(Ch);
3489 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3490 S.getIterationVariable()->getType(),
3491 S.getBeginLoc());
3492 }
3493 } else {
3494 // Default behaviour for dist_schedule clause.
3495 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3496 *this, S, ScheduleKind, Chunk);
3497 }
3498 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3499 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3500
3501 // OpenMP [2.10.8, distribute Construct, Description]
3502 // If dist_schedule is specified, kind must be static. If specified,
3503 // iterations are divided into chunks of size chunk_size, chunks are
3504 // assigned to the teams of the league in a round-robin fashion in the
3505 // order of the team number. When no chunk_size is specified, the
3506 // iteration space is divided into chunks that are approximately equal
3507 // in size, and at most one chunk is distributed to each team of the
3508 // league. The size of the chunks is unspecified in this case.
3509 bool StaticChunked = RT.isStaticChunked(
3510 ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3511 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3512 if (RT.isStaticNonchunked(ScheduleKind,
3513 /* Chunked */ Chunk != nullptr) ||
3514 StaticChunked) {
3515 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3516 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3517 CGOpenMPRuntime::StaticRTInput StaticInit(
3518 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3519 LB.getAddress(), UB.getAddress(), ST.getAddress(),
3520 StaticChunked ? Chunk : nullptr);
3521 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3522 StaticInit);
3523 JumpDest LoopExit =
3524 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3525 // UB = min(UB, GlobalUB);
3526 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3527 ? S.getCombinedEnsureUpperBound()
3528 : S.getEnsureUpperBound());
3529 // IV = LB;
3530 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3531 ? S.getCombinedInit()
3532 : S.getInit());
3533
3534 const Expr *Cond =
3535 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3536 ? S.getCombinedCond()
3537 : S.getCond();
3538
3539 if (StaticChunked)
3540 Cond = S.getCombinedDistCond();
3541
3542 // For static unchunked schedules generate:
3543 //
3544 // 1. For distribute alone, codegen
3545 // while (idx <= UB) {
3546 // BODY;
3547 // ++idx;
3548 // }
3549 //
3550 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3551 // while (idx <= UB) {
3552 // <CodeGen rest of pragma>(LB, UB);
3553 // idx += ST;
3554 // }
3555 //
3556 // For static chunk one schedule generate:
3557 //
3558 // while (IV <= GlobalUB) {
3559 // <CodeGen rest of pragma>(LB, UB);
3560 // LB += ST;
3561 // UB += ST;
3562 // UB = min(UB, GlobalUB);
3563 // IV = LB;
3564 // }
3565 //
3566 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3567 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3568 CodeGenLoop(CGF, S, LoopExit);
3569 },
3570 [&S, StaticChunked](CodeGenFunction &CGF) {
3571 if (StaticChunked) {
3572 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3573 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3574 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3575 CGF.EmitIgnoredExpr(S.getCombinedInit());
3576 }
3577 });
3578 EmitBlock(LoopExit.getBlock());
3579 // Tell the runtime we are done.
3580 RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3581 } else {
3582 // Emit the outer loop, which requests its work chunk [LB..UB] from
3583 // runtime and runs the inner loop to process it.
3584 const OMPLoopArguments LoopArguments = {
3585 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3586 Chunk};
3587 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3588 CodeGenLoop);
3589 }
3590 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3591 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3592 return CGF.Builder.CreateIsNotNull(
3593 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3594 });
3595 }
3596 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3597 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3598 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
3599 EmitOMPReductionClauseFinal(S, OMPD_simd);
3600 // Emit post-update of the reduction variables if IsLastIter != 0.
3601 emitPostUpdateForReductionClause(
3602 *this, S, [IL, &S](CodeGenFunction &CGF) {
3603 return CGF.Builder.CreateIsNotNull(
3604 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3605 });
3606 }
3607 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3608 if (HasLastprivateClause) {
3609 EmitOMPLastprivateClauseFinal(
3610 S, /*NoFinals=*/false,
3611 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3612 }
3613 }
3614
3615 // We're now done with the loop, so jump to the continuation block.
3616 if (ContBlock) {
3617 EmitBranch(ContBlock);
3618 EmitBlock(ContBlock, true);
3619 }
3620 }
3621}
3622
3623void CodeGenFunction::EmitOMPDistributeDirective(
3624 const OMPDistributeDirective &S) {
3625 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3626 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3627 };
3628 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3629 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3630}
3631
3632static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3633 const CapturedStmt *S) {
3634 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3635 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3636 CGF.CapturedStmtInfo = &CapStmtInfo;
3637 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3638 Fn->setDoesNotRecurse();
3639 return Fn;
3640}
3641
3642void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3643 if (S.hasClausesOfKind<OMPDependClause>()) {
3644 assert(!S.getAssociatedStmt() &&((!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct."
) ? static_cast<void> (0) : __assert_fail ("!S.getAssociatedStmt() && \"No associated statement must be in ordered depend construct.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3645, __PRETTY_FUNCTION__))
3645 "No associated statement must be in ordered depend construct.")((!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct."
) ? static_cast<void> (0) : __assert_fail ("!S.getAssociatedStmt() && \"No associated statement must be in ordered depend construct.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3645, __PRETTY_FUNCTION__))
;
3646 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3647 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3648 return;
3649 }
3650 const auto *C = S.getSingleClause<OMPSIMDClause>();
3651 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3652 PrePostActionTy &Action) {
3653 const CapturedStmt *CS = S.getInnermostCapturedStmt();
3654 if (C) {
3655 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3656 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3657 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3658 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3659 OutlinedFn, CapturedVars);
3660 } else {
3661 Action.Enter(CGF);
3662 CGF.EmitStmt(CS->getCapturedStmt());
3663 }
3664 };
3665 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3666 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3667}
3668
3669static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3670 QualType SrcType, QualType DestType,
3671 SourceLocation Loc) {
3672 assert(CGF.hasScalarEvaluationKind(DestType) &&((CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind."
) ? static_cast<void> (0) : __assert_fail ("CGF.hasScalarEvaluationKind(DestType) && \"DestType must have scalar evaluation kind.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3673, __PRETTY_FUNCTION__))
3673 "DestType must have scalar evaluation kind.")((CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind."
) ? static_cast<void> (0) : __assert_fail ("CGF.hasScalarEvaluationKind(DestType) && \"DestType must have scalar evaluation kind.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3673, __PRETTY_FUNCTION__))
;
3674 assert(!Val.isAggregate() && "Must be a scalar or complex.")((!Val.isAggregate() && "Must be a scalar or complex."
) ? static_cast<void> (0) : __assert_fail ("!Val.isAggregate() && \"Must be a scalar or complex.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3674, __PRETTY_FUNCTION__))
;
3675 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3676 DestType, Loc)
3677 : CGF.EmitComplexToScalarConversion(
3678 Val.getComplexVal(), SrcType, DestType, Loc);
3679}
3680
3681static CodeGenFunction::ComplexPairTy
3682convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3683 QualType DestType, SourceLocation Loc) {
3684 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&((CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind."
) ? static_cast<void> (0) : __assert_fail ("CGF.getEvaluationKind(DestType) == TEK_Complex && \"DestType must have complex evaluation kind.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3685, __PRETTY_FUNCTION__))
3685 "DestType must have complex evaluation kind.")((CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind."
) ? static_cast<void> (0) : __assert_fail ("CGF.getEvaluationKind(DestType) == TEK_Complex && \"DestType must have complex evaluation kind.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3685, __PRETTY_FUNCTION__))
;
3686 CodeGenFunction::ComplexPairTy ComplexVal;
3687 if (Val.isScalar()) {
3688 // Convert the input element to the element type of the complex.
3689 QualType DestElementType =
3690 DestType->castAs<ComplexType>()->getElementType();
3691 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3692 Val.getScalarVal(), SrcType, DestElementType, Loc);
3693 ComplexVal = CodeGenFunction::ComplexPairTy(
3694 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3695 } else {
3696 assert(Val.isComplex() && "Must be a scalar or complex.")((Val.isComplex() && "Must be a scalar or complex.") ?
static_cast<void> (0) : __assert_fail ("Val.isComplex() && \"Must be a scalar or complex.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3696, __PRETTY_FUNCTION__))
;
3697 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3698 QualType DestElementType =
3699 DestType->castAs<ComplexType>()->getElementType();
3700 ComplexVal.first = CGF.EmitScalarConversion(
3701 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3702 ComplexVal.second = CGF.EmitScalarConversion(
3703 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3704 }
3705 return ComplexVal;
3706}
3707
3708static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3709 LValue LVal, RValue RVal) {
3710 if (LVal.isGlobalReg()) {
3711 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3712 } else {
3713 CGF.EmitAtomicStore(RVal, LVal,
3714 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3715 : llvm::AtomicOrdering::Monotonic,
3716 LVal.isVolatile(), /*isInit=*/false);
3717 }
3718}
3719
3720void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3721 QualType RValTy, SourceLocation Loc) {
3722 switch (getEvaluationKind(LVal.getType())) {
3723 case TEK_Scalar:
3724 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3725 *this, RVal, RValTy, LVal.getType(), Loc)),
3726 LVal);
3727 break;
3728 case TEK_Complex:
3729 EmitStoreOfComplex(
3730 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3731 /*isInit=*/false);
3732 break;
3733 case TEK_Aggregate:
3734 llvm_unreachable("Must be a scalar or complex.")::llvm::llvm_unreachable_internal("Must be a scalar or complex."
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3734)
;
3735 }
3736}
3737
3738static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3739 const Expr *X, const Expr *V,
3740 SourceLocation Loc) {
3741 // v = x;
3742 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue")((V->isLValue() && "V of 'omp atomic read' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("V->isLValue() && \"V of 'omp atomic read' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3742, __PRETTY_FUNCTION__))
;
3743 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue")((X->isLValue() && "X of 'omp atomic read' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic read' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3743, __PRETTY_FUNCTION__))
;
3744 LValue XLValue = CGF.EmitLValue(X);
3745 LValue VLValue = CGF.EmitLValue(V);
3746 RValue Res = XLValue.isGlobalReg()
3747 ? CGF.EmitLoadOfLValue(XLValue, Loc)
3748 : CGF.EmitAtomicLoad(
3749 XLValue, Loc,
3750 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3751 : llvm::AtomicOrdering::Monotonic,
3752 XLValue.isVolatile());
3753 // OpenMP, 2.12.6, atomic Construct
3754 // Any atomic construct with a seq_cst clause forces the atomically
3755 // performed operation to include an implicit flush operation without a
3756 // list.
3757 if (IsSeqCst)
3758 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3759 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3760}
3761
3762static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3763 const Expr *X, const Expr *E,
3764 SourceLocation Loc) {
3765 // x = expr;
3766 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue")((X->isLValue() && "X of 'omp atomic write' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic write' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3766, __PRETTY_FUNCTION__))
;
3767 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3768 // OpenMP, 2.12.6, atomic Construct
3769 // Any atomic construct with a seq_cst clause forces the atomically
3770 // performed operation to include an implicit flush operation without a
3771 // list.
3772 if (IsSeqCst)
3773 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3774}
3775
3776static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3777 RValue Update,
3778 BinaryOperatorKind BO,
3779 llvm::AtomicOrdering AO,
3780 bool IsXLHSInRHSPart) {
3781 ASTContext &Context = CGF.getContext();
3782 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3783 // expression is simple and atomic is allowed for the given type for the
3784 // target platform.
3785 if (BO == BO_Comma || !Update.isScalar() ||
3786 !Update.getScalarVal()->getType()->isIntegerTy() ||
3787 !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3788 (Update.getScalarVal()->getType() !=
3789 X.getAddress().getElementType())) ||
3790 !X.getAddress().getElementType()->isIntegerTy() ||
3791 !Context.getTargetInfo().hasBuiltinAtomic(
3792 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3793 return std::make_pair(false, RValue::get(nullptr));
3794
3795 llvm::AtomicRMWInst::BinOp RMWOp;
3796 switch (BO) {
3797 case BO_Add:
3798 RMWOp = llvm::AtomicRMWInst::Add;
3799 break;
3800 case BO_Sub:
3801 if (!IsXLHSInRHSPart)
3802 return std::make_pair(false, RValue::get(nullptr));
3803 RMWOp = llvm::AtomicRMWInst::Sub;
3804 break;
3805 case BO_And:
3806 RMWOp = llvm::AtomicRMWInst::And;
3807 break;
3808 case BO_Or:
3809 RMWOp = llvm::AtomicRMWInst::Or;
3810 break;
3811 case BO_Xor:
3812 RMWOp = llvm::AtomicRMWInst::Xor;
3813 break;
3814 case BO_LT:
3815 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3816 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3817 : llvm::AtomicRMWInst::Max)
3818 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3819 : llvm::AtomicRMWInst::UMax);
3820 break;
3821 case BO_GT:
3822 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3823 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3824 : llvm::AtomicRMWInst::Min)
3825 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3826 : llvm::AtomicRMWInst::UMin);
3827 break;
3828 case BO_Assign:
3829 RMWOp = llvm::AtomicRMWInst::Xchg;
3830 break;
3831 case BO_Mul:
3832 case BO_Div:
3833 case BO_Rem:
3834 case BO_Shl:
3835 case BO_Shr:
3836 case BO_LAnd:
3837 case BO_LOr:
3838 return std::make_pair(false, RValue::get(nullptr));
3839 case BO_PtrMemD:
3840 case BO_PtrMemI:
3841 case BO_LE:
3842 case BO_GE:
3843 case BO_EQ:
3844 case BO_NE:
3845 case BO_Cmp:
3846 case BO_AddAssign:
3847 case BO_SubAssign:
3848 case BO_AndAssign:
3849 case BO_OrAssign:
3850 case BO_XorAssign:
3851 case BO_MulAssign:
3852 case BO_DivAssign:
3853 case BO_RemAssign:
3854 case BO_ShlAssign:
3855 case BO_ShrAssign:
3856 case BO_Comma:
3857 llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3857)
;
3858 }
3859 llvm::Value *UpdateVal = Update.getScalarVal();
3860 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3861 UpdateVal = CGF.Builder.CreateIntCast(
3862 IC, X.getAddress().getElementType(),
3863 X.getType()->hasSignedIntegerRepresentation());
3864 }
3865 llvm::Value *Res =
3866 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3867 return std::make_pair(true, RValue::get(Res));
3868}
3869
3870std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
3871 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3872 llvm::AtomicOrdering AO, SourceLocation Loc,
3873 const llvm::function_ref<RValue(RValue)> CommonGen) {
3874 // Update expressions are allowed to have the following forms:
3875 // x binop= expr; -> xrval + expr;
3876 // x++, ++x -> xrval + 1;
3877 // x--, --x -> xrval - 1;
3878 // x = x binop expr; -> xrval binop expr
3879 // x = expr Op x; - > expr binop xrval;
3880 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3881 if (!Res.first) {
3882 if (X.isGlobalReg()) {
3883 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3884 // 'xrval'.
3885 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3886 } else {
3887 // Perform compare-and-swap procedure.
3888 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3889 }
3890 }
3891 return Res;
3892}
3893
3894static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3895 const Expr *X, const Expr *E,
3896 const Expr *UE, bool IsXLHSInRHSPart,
3897 SourceLocation Loc) {
3898 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&((isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
"Update expr in 'atomic update' must be a binary operator.")
? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic update' must be a binary operator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3899, __PRETTY_FUNCTION__))
3899 "Update expr in 'atomic update' must be a binary operator.")((isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
"Update expr in 'atomic update' must be a binary operator.")
? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic update' must be a binary operator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3899, __PRETTY_FUNCTION__))
;
3900 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3901 // Update expressions are allowed to have the following forms:
3902 // x binop= expr; -> xrval + expr;
3903 // x++, ++x -> xrval + 1;
3904 // x--, --x -> xrval - 1;
3905 // x = x binop expr; -> xrval binop expr
3906 // x = expr Op x; - > expr binop xrval;
3907 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue")((X->isLValue() && "X of 'omp atomic update' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic update' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3907, __PRETTY_FUNCTION__))
;
3908 LValue XLValue = CGF.EmitLValue(X);
3909 RValue ExprRValue = CGF.EmitAnyExpr(E);
3910 llvm::AtomicOrdering AO = IsSeqCst
3911 ? llvm::AtomicOrdering::SequentiallyConsistent
3912 : llvm::AtomicOrdering::Monotonic;
3913 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3914 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3915 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3916 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3917 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3918 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3919 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3920 return CGF.EmitAnyExpr(UE);
3921 };
3922 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3923 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3924 // OpenMP, 2.12.6, atomic Construct
3925 // Any atomic construct with a seq_cst clause forces the atomically
3926 // performed operation to include an implicit flush operation without a
3927 // list.
3928 if (IsSeqCst)
3929 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3930}
3931
3932static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3933 QualType SourceType, QualType ResType,
3934 SourceLocation Loc) {
3935 switch (CGF.getEvaluationKind(ResType)) {
3936 case TEK_Scalar:
3937 return RValue::get(
3938 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3939 case TEK_Complex: {
3940 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3941 return RValue::getComplex(Res.first, Res.second);
3942 }
3943 case TEK_Aggregate:
3944 break;
3945 }
3946 llvm_unreachable("Must be a scalar or complex.")::llvm::llvm_unreachable_internal("Must be a scalar or complex."
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3946)
;
3947}
3948
3949static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3950 bool IsPostfixUpdate, const Expr *V,
3951 const Expr *X, const Expr *E,
3952 const Expr *UE, bool IsXLHSInRHSPart,
3953 SourceLocation Loc) {
3954 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue")((X->isLValue() && "X of 'omp atomic capture' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic capture' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3954, __PRETTY_FUNCTION__))
;
3955 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue")((V->isLValue() && "V of 'omp atomic capture' is not lvalue"
) ? static_cast<void> (0) : __assert_fail ("V->isLValue() && \"V of 'omp atomic capture' is not lvalue\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3955, __PRETTY_FUNCTION__))
;
3956 RValue NewVVal;
3957 LValue VLValue = CGF.EmitLValue(V);
3958 LValue XLValue = CGF.EmitLValue(X);
3959 RValue ExprRValue = CGF.EmitAnyExpr(E);
3960 llvm::AtomicOrdering AO = IsSeqCst
3961 ? llvm::AtomicOrdering::SequentiallyConsistent
3962 : llvm::AtomicOrdering::Monotonic;
3963 QualType NewVValType;
3964 if (UE) {
3965 // 'x' is updated with some additional value.
3966 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&((isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
"Update expr in 'atomic capture' must be a binary operator."
) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic capture' must be a binary operator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3967, __PRETTY_FUNCTION__))
3967 "Update expr in 'atomic capture' must be a binary operator.")((isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
"Update expr in 'atomic capture' must be a binary operator."
) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic capture' must be a binary operator.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 3967, __PRETTY_FUNCTION__))
;
3968 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3969 // Update expressions are allowed to have the following forms:
3970 // x binop= expr; -> xrval + expr;
3971 // x++, ++x -> xrval + 1;
3972 // x--, --x -> xrval - 1;
3973 // x = x binop expr; -> xrval binop expr
3974 // x = expr Op x; - > expr binop xrval;
3975 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3976 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3977 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3978 NewVValType = XRValExpr->getType();
3979 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3980 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3981 IsPostfixUpdate](RValue XRValue) {
3982 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3983 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3984 RValue Res = CGF.EmitAnyExpr(UE);
3985 NewVVal = IsPostfixUpdate ? XRValue : Res;
3986 return Res;
3987 };
3988 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3989 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3990 if (Res.first) {
3991 // 'atomicrmw' instruction was generated.
3992 if (IsPostfixUpdate) {
3993 // Use old value from 'atomicrmw'.
3994 NewVVal = Res.second;
3995 } else {
3996 // 'atomicrmw' does not provide new value, so evaluate it using old
3997 // value of 'x'.
3998 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3999 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
4000 NewVVal = CGF.EmitAnyExpr(UE);
4001 }
4002 }
4003 } else {
4004 // 'x' is simply rewritten with some 'expr'.
4005 NewVValType = X->getType().getNonReferenceType();
4006 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
4007 X->getType().getNonReferenceType(), Loc);
4008 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
4009 NewVVal = XRValue;
4010 return ExprRValue;
4011 };
4012 // Try to perform atomicrmw xchg, otherwise simple exchange.
4013 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
4014 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
4015 Loc, Gen);
4016 if (Res.first) {
4017 // 'atomicrmw' instruction was generated.
4018 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
4019 }
4020 }
4021 // Emit post-update store to 'v' of old/new 'x' value.
4022 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
4023 // OpenMP, 2.12.6, atomic Construct
4024 // Any atomic construct with a seq_cst clause forces the atomically
4025 // performed operation to include an implicit flush operation without a
4026 // list.
4027 if (IsSeqCst)
4028 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
4029}
4030
4031static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
4032 bool IsSeqCst, bool IsPostfixUpdate,
4033 const Expr *X, const Expr *V, const Expr *E,
4034 const Expr *UE, bool IsXLHSInRHSPart,
4035 SourceLocation Loc) {
4036 switch (Kind) {
4037 case OMPC_read:
4038 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
4039 break;
4040 case OMPC_write:
4041 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
4042 break;
4043 case OMPC_unknown:
4044 case OMPC_update:
4045 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
4046 break;
4047 case OMPC_capture:
4048 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
4049 IsXLHSInRHSPart, Loc);
4050 break;
4051 case OMPC_if:
4052 case OMPC_final:
4053 case OMPC_num_threads:
4054 case OMPC_private:
4055 case OMPC_firstprivate:
4056 case OMPC_lastprivate:
4057 case OMPC_reduction:
4058 case OMPC_task_reduction:
4059 case OMPC_in_reduction:
4060 case OMPC_safelen:
4061 case OMPC_simdlen:
4062 case OMPC_allocator:
4063 case OMPC_allocate:
4064 case OMPC_collapse:
4065 case OMPC_default:
4066 case OMPC_seq_cst:
4067 case OMPC_shared:
4068 case OMPC_linear:
4069 case OMPC_aligned:
4070 case OMPC_copyin:
4071 case OMPC_copyprivate:
4072 case OMPC_flush:
4073 case OMPC_proc_bind:
4074 case OMPC_schedule:
4075 case OMPC_ordered:
4076 case OMPC_nowait:
4077 case OMPC_untied:
4078 case OMPC_threadprivate:
4079 case OMPC_depend:
4080 case OMPC_mergeable:
4081 case OMPC_device:
4082 case OMPC_threads:
4083 case OMPC_simd:
4084 case OMPC_map:
4085 case OMPC_num_teams:
4086 case OMPC_thread_limit:
4087 case OMPC_priority:
4088 case OMPC_grainsize:
4089 case OMPC_nogroup:
4090 case OMPC_num_tasks:
4091 case OMPC_hint:
4092 case OMPC_dist_schedule:
4093 case OMPC_defaultmap:
4094 case OMPC_uniform:
4095 case OMPC_to:
4096 case OMPC_from:
4097 case OMPC_use_device_ptr:
4098 case OMPC_is_device_ptr:
4099 case OMPC_unified_address:
4100 case OMPC_unified_shared_memory:
4101 case OMPC_reverse_offload:
4102 case OMPC_dynamic_allocators:
4103 case OMPC_atomic_default_mem_order:
4104 case OMPC_device_type:
4105 case OMPC_match:
4106 llvm_unreachable("Clause is not allowed in 'omp atomic'.")::llvm::llvm_unreachable_internal("Clause is not allowed in 'omp atomic'."
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4106)
;
4107 }
4108}
4109
4110void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
4111 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4112 OpenMPClauseKind Kind = OMPC_unknown;
4113 for (const OMPClause *C : S.clauses()) {
4114 // Find first clause (skip seq_cst clause, if it is first).
4115 if (C->getClauseKind() != OMPC_seq_cst) {
4116 Kind = C->getClauseKind();
4117 break;
4118 }
4119 }
4120
4121 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4122 if (const auto *FE = dyn_cast<FullExpr>(CS))
4123 enterFullExpression(FE);
4124 // Processing for statements under 'atomic capture'.
4125 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4126 for (const Stmt *C : Compound->body()) {
4127 if (const auto *FE = dyn_cast<FullExpr>(C))
4128 enterFullExpression(FE);
4129 }
4130 }
4131
4132 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4133 PrePostActionTy &) {
4134 CGF.EmitStopPoint(CS);
4135 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4136 S.getV(), S.getExpr(), S.getUpdateExpr(),
4137 S.isXLHSInRHSPart(), S.getBeginLoc());
4138 };
4139 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4140 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4141}
4142
4143static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
4144 const OMPExecutableDirective &S,
4145 const RegionCodeGenTy &CodeGen) {
4146 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()))((isOpenMPTargetExecutionDirective(S.getDirectiveKind())) ? static_cast
<void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(S.getDirectiveKind())"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4146, __PRETTY_FUNCTION__))
;
4147 CodeGenModule &CGM = CGF.CGM;
4148
4149 // On device emit this construct as inlined code.
4150 if (CGM.getLangOpts().OpenMPIsDevice) {
4151 OMPLexicalScope Scope(CGF, S, OMPD_target);
4152 CGM.getOpenMPRuntime().emitInlinedDirective(
4153 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4154 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4155 });
4156 return;
4157 }
4158
4159 llvm::Function *Fn = nullptr;
4160 llvm::Constant *FnID = nullptr;
4161
4162 const Expr *IfCond = nullptr;
4163 // Check for the at most one if clause associated with the target region.
4164 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4165 if (C->getNameModifier() == OMPD_unknown ||
4166 C->getNameModifier() == OMPD_target) {
4167 IfCond = C->getCondition();
4168 break;
4169 }
4170 }
4171
4172 // Check if we have any device clause associated with the directive.
4173 const Expr *Device = nullptr;
4174 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4175 Device = C->getDevice();
4176
4177 // Check if we have an if clause whose conditional always evaluates to false
4178 // or if we do not have any targets specified. If so the target region is not
4179 // an offload entry point.
4180 bool IsOffloadEntry = true;
4181 if (IfCond) {
4182 bool Val;
4183 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4184 IsOffloadEntry = false;
4185 }
4186 if (CGM.getLangOpts().OMPTargetTriples.empty())
4187 IsOffloadEntry = false;
4188
4189 assert(CGF.CurFuncDecl && "No parent declaration for target region!")((CGF.CurFuncDecl && "No parent declaration for target region!"
) ? static_cast<void> (0) : __assert_fail ("CGF.CurFuncDecl && \"No parent declaration for target region!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4189, __PRETTY_FUNCTION__))
;
4190 StringRef ParentName;
4191 // In case we have Ctors/Dtors we use the complete type variant to produce
4192 // the mangling of the device outlined kernel.
4193 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4194 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4195 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4196 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4197 else
4198 ParentName =
4199 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4200
4201 // Emit target region as a standalone region.
4202 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4203 IsOffloadEntry, CodeGen);
4204 OMPLexicalScope Scope(CGF, S, OMPD_task);
4205 auto &&SizeEmitter =
4206 [IsOffloadEntry](CodeGenFunction &CGF,
4207 const OMPLoopDirective &D) -> llvm::Value * {
4208 if (IsOffloadEntry) {
4209 OMPLoopScope(CGF, D);
4210 // Emit calculation of the iterations count.
4211 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4212 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4213 /*isSigned=*/false);
4214 return NumIterations;
4215 }
4216 return nullptr;
4217 };
4218 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
4219 SizeEmitter);
4220}
4221
4222static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
4223 PrePostActionTy &Action) {
4224 Action.Enter(CGF);
4225 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4226 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4227 CGF.EmitOMPPrivateClause(S, PrivateScope);
4228 (void)PrivateScope.Privatize();
4229 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4230 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4231
4232 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4233}
4234
4235void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4236 StringRef ParentName,
4237 const OMPTargetDirective &S) {
4238 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4239 emitTargetRegion(CGF, S, Action);
4240 };
4241 llvm::Function *Fn;
4242 llvm::Constant *Addr;
4243 // Emit target region as a standalone region.
4244 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4245 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4246 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4246, __PRETTY_FUNCTION__))
;
4247}
4248
4249void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4250 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4251 emitTargetRegion(CGF, S, Action);
4252 };
4253 emitCommonOMPTargetDirective(*this, S, CodeGen);
4254}
4255
4256static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4257 const OMPExecutableDirective &S,
4258 OpenMPDirectiveKind InnermostKind,
4259 const RegionCodeGenTy &CodeGen) {
4260 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4261 llvm::Function *OutlinedFn =
4262 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4263 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4264
4265 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4266 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4267 if (NT || TL) {
4268 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4269 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4270
4271 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4272 S.getBeginLoc());
4273 }
4274
4275 OMPTeamsScope Scope(CGF, S);
4276 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4277 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4278 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4279 CapturedVars);
4280}
4281
4282void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
4283 // Emit teams region as a standalone region.
4284 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4285 Action.Enter(CGF);
4286 OMPPrivateScope PrivateScope(CGF);
4287 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4288 CGF.EmitOMPPrivateClause(S, PrivateScope);
4289 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4290 (void)PrivateScope.Privatize();
4291 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4292 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4293 };
4294 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4295 emitPostUpdateForReductionClause(*this, S,
4296 [](CodeGenFunction &) { return nullptr; });
4297}
4298
4299static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4300 const OMPTargetTeamsDirective &S) {
4301 auto *CS = S.getCapturedStmt(OMPD_teams);
4302 Action.Enter(CGF);
4303 // Emit teams region as a standalone region.
4304 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4305 Action.Enter(CGF);
4306 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4307 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4308 CGF.EmitOMPPrivateClause(S, PrivateScope);
4309 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4310 (void)PrivateScope.Privatize();
4311 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4312 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4313 CGF.EmitStmt(CS->getCapturedStmt());
4314 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4315 };
4316 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4317 emitPostUpdateForReductionClause(CGF, S,
4318 [](CodeGenFunction &) { return nullptr; });
4319}
4320
4321void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4322 CodeGenModule &CGM, StringRef ParentName,
4323 const OMPTargetTeamsDirective &S) {
4324 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4325 emitTargetTeamsRegion(CGF, Action, S);
4326 };
4327 llvm::Function *Fn;
4328 llvm::Constant *Addr;
4329 // Emit target region as a standalone region.
4330 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4331 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4332 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4332, __PRETTY_FUNCTION__))
;
4333}
4334
4335void CodeGenFunction::EmitOMPTargetTeamsDirective(
4336 const OMPTargetTeamsDirective &S) {
4337 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4338 emitTargetTeamsRegion(CGF, Action, S);
4339 };
4340 emitCommonOMPTargetDirective(*this, S, CodeGen);
4341}
4342
4343static void
4344emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4345 const OMPTargetTeamsDistributeDirective &S) {
4346 Action.Enter(CGF);
4347 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4348 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4349 };
4350
4351 // Emit teams region as a standalone region.
4352 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4353 PrePostActionTy &Action) {
4354 Action.Enter(CGF);
4355 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4356 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4357 (void)PrivateScope.Privatize();
4358 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4359 CodeGenDistribute);
4360 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4361 };
4362 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4363 emitPostUpdateForReductionClause(CGF, S,
4364 [](CodeGenFunction &) { return nullptr; });
4365}
4366
4367void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4368 CodeGenModule &CGM, StringRef ParentName,
4369 const OMPTargetTeamsDistributeDirective &S) {
4370 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4371 emitTargetTeamsDistributeRegion(CGF, Action, S);
4372 };
4373 llvm::Function *Fn;
4374 llvm::Constant *Addr;
4375 // Emit target region as a standalone region.
4376 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4377 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4378 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4378, __PRETTY_FUNCTION__))
;
4379}
4380
4381void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4382 const OMPTargetTeamsDistributeDirective &S) {
4383 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4384 emitTargetTeamsDistributeRegion(CGF, Action, S);
4385 };
4386 emitCommonOMPTargetDirective(*this, S, CodeGen);
4387}
4388
4389static void emitTargetTeamsDistributeSimdRegion(
4390 CodeGenFunction &CGF, PrePostActionTy &Action,
4391 const OMPTargetTeamsDistributeSimdDirective &S) {
4392 Action.Enter(CGF);
4393 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4394 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4395 };
4396
4397 // Emit teams region as a standalone region.
4398 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4399 PrePostActionTy &Action) {
4400 Action.Enter(CGF);
4401 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4402 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4403 (void)PrivateScope.Privatize();
4404 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4405 CodeGenDistribute);
4406 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4407 };
4408 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4409 emitPostUpdateForReductionClause(CGF, S,
4410 [](CodeGenFunction &) { return nullptr; });
4411}
4412
4413void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4414 CodeGenModule &CGM, StringRef ParentName,
4415 const OMPTargetTeamsDistributeSimdDirective &S) {
4416 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4417 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4418 };
4419 llvm::Function *Fn;
4420 llvm::Constant *Addr;
4421 // Emit target region as a standalone region.
4422 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4423 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4424 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4424, __PRETTY_FUNCTION__))
;
4425}
4426
4427void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4428 const OMPTargetTeamsDistributeSimdDirective &S) {
4429 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4430 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4431 };
4432 emitCommonOMPTargetDirective(*this, S, CodeGen);
4433}
4434
4435void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4436 const OMPTeamsDistributeDirective &S) {
4437
4438 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4439 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4440 };
4441
4442 // Emit teams region as a standalone region.
4443 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4444 PrePostActionTy &Action) {
4445 Action.Enter(CGF);
4446 OMPPrivateScope PrivateScope(CGF);
4447 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4448 (void)PrivateScope.Privatize();
4449 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4450 CodeGenDistribute);
4451 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4452 };
4453 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4454 emitPostUpdateForReductionClause(*this, S,
4455 [](CodeGenFunction &) { return nullptr; });
4456}
4457
4458void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4459 const OMPTeamsDistributeSimdDirective &S) {
4460 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4461 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4462 };
4463
4464 // Emit teams region as a standalone region.
4465 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4466 PrePostActionTy &Action) {
4467 Action.Enter(CGF);
4468 OMPPrivateScope PrivateScope(CGF);
4469 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4470 (void)PrivateScope.Privatize();
4471 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4472 CodeGenDistribute);
4473 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4474 };
4475 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4476 emitPostUpdateForReductionClause(*this, S,
4477 [](CodeGenFunction &) { return nullptr; });
4478}
4479
4480void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4481 const OMPTeamsDistributeParallelForDirective &S) {
4482 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4483 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4484 S.getDistInc());
4485 };
4486
4487 // Emit teams region as a standalone region.
4488 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4489 PrePostActionTy &Action) {
4490 Action.Enter(CGF);
4491 OMPPrivateScope PrivateScope(CGF);
4492 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4493 (void)PrivateScope.Privatize();
4494 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4495 CodeGenDistribute);
4496 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4497 };
4498 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4499 emitPostUpdateForReductionClause(*this, S,
4500 [](CodeGenFunction &) { return nullptr; });
4501}
4502
4503void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4504 const OMPTeamsDistributeParallelForSimdDirective &S) {
4505 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4506 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4507 S.getDistInc());
4508 };
4509
4510 // Emit teams region as a standalone region.
4511 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4512 PrePostActionTy &Action) {
4513 Action.Enter(CGF);
4514 OMPPrivateScope PrivateScope(CGF);
4515 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4516 (void)PrivateScope.Privatize();
4517 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4518 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4519 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4520 };
4521 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4522 emitPostUpdateForReductionClause(*this, S,
4523 [](CodeGenFunction &) { return nullptr; });
4524}
4525
4526static void emitTargetTeamsDistributeParallelForRegion(
4527 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
4528 PrePostActionTy &Action) {
4529 Action.Enter(CGF);
4530 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4531 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4532 S.getDistInc());
4533 };
4534
4535 // Emit teams region as a standalone region.
4536 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4537 PrePostActionTy &Action) {
4538 Action.Enter(CGF);
4539 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4540 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4541 (void)PrivateScope.Privatize();
4542 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4543 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4544 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4545 };
4546
4547 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4548 CodeGenTeams);
4549 emitPostUpdateForReductionClause(CGF, S,
4550 [](CodeGenFunction &) { return nullptr; });
4551}
4552
4553void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4554 CodeGenModule &CGM, StringRef ParentName,
4555 const OMPTargetTeamsDistributeParallelForDirective &S) {
4556 // Emit SPMD target teams distribute parallel for region as a standalone
4557 // region.
4558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4559 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4560 };
4561 llvm::Function *Fn;
4562 llvm::Constant *Addr;
4563 // Emit target region as a standalone region.
4564 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4565 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4566 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4566, __PRETTY_FUNCTION__))
;
4567}
4568
4569void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
4570 const OMPTargetTeamsDistributeParallelForDirective &S) {
4571 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4572 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4573 };
4574 emitCommonOMPTargetDirective(*this, S, CodeGen);
4575}
4576
4577static void emitTargetTeamsDistributeParallelForSimdRegion(
4578 CodeGenFunction &CGF,
4579 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
4580 PrePostActionTy &Action) {
4581 Action.Enter(CGF);
4582 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4583 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4584 S.getDistInc());
4585 };
4586
4587 // Emit teams region as a standalone region.
4588 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4589 PrePostActionTy &Action) {
4590 Action.Enter(CGF);
4591 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4592 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4593 (void)PrivateScope.Privatize();
4594 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4595 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4596 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4597 };
4598
4599 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4600 CodeGenTeams);
4601 emitPostUpdateForReductionClause(CGF, S,
4602 [](CodeGenFunction &) { return nullptr; });
4603}
4604
4605void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4606 CodeGenModule &CGM, StringRef ParentName,
4607 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4608 // Emit SPMD target teams distribute parallel for simd region as a standalone
4609 // region.
4610 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4611 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4612 };
4613 llvm::Function *Fn;
4614 llvm::Constant *Addr;
4615 // Emit target region as a standalone region.
4616 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4617 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4618 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4618, __PRETTY_FUNCTION__))
;
4619}
4620
4621void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4622 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4623 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4624 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4625 };
4626 emitCommonOMPTargetDirective(*this, S, CodeGen);
4627}
4628
4629void CodeGenFunction::EmitOMPCancellationPointDirective(
4630 const OMPCancellationPointDirective &S) {
4631 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4632 S.getCancelRegion());
4633}
4634
4635void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
4636 const Expr *IfCond = nullptr;
4637 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4638 if (C->getNameModifier() == OMPD_unknown ||
4639 C->getNameModifier() == OMPD_cancel) {
4640 IfCond = C->getCondition();
4641 break;
4642 }
4643 }
4644 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4645 S.getCancelRegion());
4646}
4647
4648CodeGenFunction::JumpDest
4649CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
4650 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4651 Kind == OMPD_target_parallel)
4652 return ReturnBlock;
4653 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
4654 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
4655 Kind == OMPD_distribute_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
4656 Kind == OMPD_target_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
4657 Kind == OMPD_teams_distribute_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
4658 Kind == OMPD_target_teams_distribute_parallel_for)((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections
|| Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for
|| Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for
|| Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for
) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4658, __PRETTY_FUNCTION__))
;
4659 return OMPCancelStack.getExitBlock();
4660}
4661
4662void CodeGenFunction::EmitOMPUseDevicePtrClause(
4663 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4664 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4665 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4666 auto OrigVarIt = C.varlist_begin();
4667 auto InitIt = C.inits().begin();
4668 for (const Expr *PvtVarIt : C.private_copies()) {
4669 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4670 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4671 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4672
4673 // In order to identify the right initializer we need to match the
4674 // declaration used by the mapping logic. In some cases we may get
4675 // OMPCapturedExprDecl that refers to the original declaration.
4676 const ValueDecl *MatchingVD = OrigVD;
4677 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4678 // OMPCapturedExprDecl are used to privative fields of the current
4679 // structure.
4680 const auto *ME = cast<MemberExpr>(OED->getInit());
4681 assert(isa<CXXThisExpr>(ME->getBase()) &&((isa<CXXThisExpr>(ME->getBase()) && "Base should be the current struct!"
) ? static_cast<void> (0) : __assert_fail ("isa<CXXThisExpr>(ME->getBase()) && \"Base should be the current struct!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4682, __PRETTY_FUNCTION__))
4682 "Base should be the current struct!")((isa<CXXThisExpr>(ME->getBase()) && "Base should be the current struct!"
) ? static_cast<void> (0) : __assert_fail ("isa<CXXThisExpr>(ME->getBase()) && \"Base should be the current struct!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4682, __PRETTY_FUNCTION__))
;
4683 MatchingVD = ME->getMemberDecl();
4684 }
4685
4686 // If we don't have information about the current list item, move on to
4687 // the next one.
4688 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4689 if (InitAddrIt == CaptureDeviceAddrMap.end())
4690 continue;
4691
4692 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4693 InitAddrIt, InitVD,
4694 PvtVD]() {
4695 // Initialize the temporary initialization variable with the address we
4696 // get from the runtime library. We have to cast the source address
4697 // because it is always a void *. References are materialized in the
4698 // privatization scope, so the initialization here disregards the fact
4699 // the original variable is a reference.
4700 QualType AddrQTy =
4701 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4702 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4703 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4704 setAddrOfLocalVar(InitVD, InitAddr);
4705
4706 // Emit private declaration, it will be initialized by the value we
4707 // declaration we just added to the local declarations map.
4708 EmitDecl(*PvtVD);
4709
4710 // The initialization variables reached its purpose in the emission
4711 // of the previous declaration, so we don't need it anymore.
4712 LocalDeclMap.erase(InitVD);
4713
4714 // Return the address of the private variable.
4715 return GetAddrOfLocalVar(PvtVD);
4716 });
4717 assert(IsRegistered && "firstprivate var already registered as private")((IsRegistered && "firstprivate var already registered as private"
) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4717, __PRETTY_FUNCTION__))
;
4718 // Silence the warning about unused variable.
4719 (void)IsRegistered;
4720
4721 ++OrigVarIt;
4722 ++InitIt;
4723 }
4724}
4725
4726// Generate the instructions for '#pragma omp target data' directive.
4727void CodeGenFunction::EmitOMPTargetDataDirective(
4728 const OMPTargetDataDirective &S) {
4729 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4730
4731 // Create a pre/post action to signal the privatization of the device pointer.
4732 // This action can be replaced by the OpenMP runtime code generation to
4733 // deactivate privatization.
4734 bool PrivatizeDevicePointers = false;
4735 class DevicePointerPrivActionTy : public PrePostActionTy {
4736 bool &PrivatizeDevicePointers;
4737
4738 public:
4739 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4740 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4741 void Enter(CodeGenFunction &CGF) override {
4742 PrivatizeDevicePointers = true;
4743 }
4744 };
4745 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4746
4747 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4748 CodeGenFunction &CGF, PrePostActionTy &Action) {
4749 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4750 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4751 };
4752
4753 // Codegen that selects whether to generate the privatization code or not.
4754 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4755 &InnermostCodeGen](CodeGenFunction &CGF,
4756 PrePostActionTy &Action) {
4757 RegionCodeGenTy RCG(InnermostCodeGen);
4758 PrivatizeDevicePointers = false;
4759
4760 // Call the pre-action to change the status of PrivatizeDevicePointers if
4761 // needed.
4762 Action.Enter(CGF);
4763
4764 if (PrivatizeDevicePointers) {
4765 OMPPrivateScope PrivateScope(CGF);
4766 // Emit all instances of the use_device_ptr clause.
4767 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4768 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4769 Info.CaptureDeviceAddrMap);
4770 (void)PrivateScope.Privatize();
4771 RCG(CGF);
4772 } else {
4773 RCG(CGF);
4774 }
4775 };
4776
4777 // Forward the provided action to the privatization codegen.
4778 RegionCodeGenTy PrivRCG(PrivCodeGen);
4779 PrivRCG.setAction(Action);
4780
4781 // Notwithstanding the body of the region is emitted as inlined directive,
4782 // we don't use an inline scope as changes in the references inside the
4783 // region are expected to be visible outside, so we do not privative them.
4784 OMPLexicalScope Scope(CGF, S);
4785 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4786 PrivRCG);
4787 };
4788
4789 RegionCodeGenTy RCG(CodeGen);
4790
4791 // If we don't have target devices, don't bother emitting the data mapping
4792 // code.
4793 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4794 RCG(*this);
4795 return;
4796 }
4797
4798 // Check if we have any if clause associated with the directive.
4799 const Expr *IfCond = nullptr;
4800 if (const auto *C = S.getSingleClause<OMPIfClause>())
4801 IfCond = C->getCondition();
4802
4803 // Check if we have any device clause associated with the directive.
4804 const Expr *Device = nullptr;
4805 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4806 Device = C->getDevice();
4807
4808 // Set the action to signal privatization of device pointers.
4809 RCG.setAction(PrivAction);
4810
4811 // Emit region code.
4812 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4813 Info);
4814}
4815
4816void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4817 const OMPTargetEnterDataDirective &S) {
4818 // If we don't have target devices, don't bother emitting the data mapping
4819 // code.
4820 if (CGM.getLangOpts().OMPTargetTriples.empty())
4821 return;
4822
4823 // Check if we have any if clause associated with the directive.
4824 const Expr *IfCond = nullptr;
4825 if (const auto *C = S.getSingleClause<OMPIfClause>())
4826 IfCond = C->getCondition();
4827
4828 // Check if we have any device clause associated with the directive.
4829 const Expr *Device = nullptr;
4830 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4831 Device = C->getDevice();
4832
4833 OMPLexicalScope Scope(*this, S, OMPD_task);
4834 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4835}
4836
4837void CodeGenFunction::EmitOMPTargetExitDataDirective(
4838 const OMPTargetExitDataDirective &S) {
4839 // If we don't have target devices, don't bother emitting the data mapping
4840 // code.
4841 if (CGM.getLangOpts().OMPTargetTriples.empty())
4842 return;
4843
4844 // Check if we have any if clause associated with the directive.
4845 const Expr *IfCond = nullptr;
4846 if (const auto *C = S.getSingleClause<OMPIfClause>())
4847 IfCond = C->getCondition();
4848
4849 // Check if we have any device clause associated with the directive.
4850 const Expr *Device = nullptr;
4851 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4852 Device = C->getDevice();
4853
4854 OMPLexicalScope Scope(*this, S, OMPD_task);
4855 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4856}
4857
4858static void emitTargetParallelRegion(CodeGenFunction &CGF,
4859 const OMPTargetParallelDirective &S,
4860 PrePostActionTy &Action) {
4861 // Get the captured statement associated with the 'parallel' region.
4862 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4863 Action.Enter(CGF);
4864 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4865 Action.Enter(CGF);
4866 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4867 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4868 CGF.EmitOMPPrivateClause(S, PrivateScope);
4869 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4870 (void)PrivateScope.Privatize();
4871 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4872 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4873 // TODO: Add support for clauses.
4874 CGF.EmitStmt(CS->getCapturedStmt());
4875 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4876 };
4877 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4878 emitEmptyBoundParameters);
4879 emitPostUpdateForReductionClause(CGF, S,
4880 [](CodeGenFunction &) { return nullptr; });
4881}
4882
4883void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4884 CodeGenModule &CGM, StringRef ParentName,
4885 const OMPTargetParallelDirective &S) {
4886 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4887 emitTargetParallelRegion(CGF, S, Action);
4888 };
4889 llvm::Function *Fn;
4890 llvm::Constant *Addr;
4891 // Emit target region as a standalone region.
4892 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4893 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4894 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4894, __PRETTY_FUNCTION__))
;
4895}
4896
4897void CodeGenFunction::EmitOMPTargetParallelDirective(
4898 const OMPTargetParallelDirective &S) {
4899 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4900 emitTargetParallelRegion(CGF, S, Action);
4901 };
4902 emitCommonOMPTargetDirective(*this, S, CodeGen);
4903}
4904
4905static void emitTargetParallelForRegion(CodeGenFunction &CGF,
4906 const OMPTargetParallelForDirective &S,
4907 PrePostActionTy &Action) {
4908 Action.Enter(CGF);
4909 // Emit directive as a combined directive that consists of two implicit
4910 // directives: 'parallel' with 'for' directive.
4911 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4912 Action.Enter(CGF);
4913 CodeGenFunction::OMPCancelStackRAII CancelRegion(
4914 CGF, OMPD_target_parallel_for, S.hasCancel());
4915 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4916 emitDispatchForLoopBounds);
4917 };
4918 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4919 emitEmptyBoundParameters);
4920}
4921
4922void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
4923 CodeGenModule &CGM, StringRef ParentName,
4924 const OMPTargetParallelForDirective &S) {
4925 // Emit SPMD target parallel for region as a standalone region.
4926 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4927 emitTargetParallelForRegion(CGF, S, Action);
4928 };
4929 llvm::Function *Fn;
4930 llvm::Constant *Addr;
4931 // Emit target region as a standalone region.
4932 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4933 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4934 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4934, __PRETTY_FUNCTION__))
;
4935}
4936
4937void CodeGenFunction::EmitOMPTargetParallelForDirective(
4938 const OMPTargetParallelForDirective &S) {
4939 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4940 emitTargetParallelForRegion(CGF, S, Action);
4941 };
4942 emitCommonOMPTargetDirective(*this, S, CodeGen);
4943}
4944
4945static void
4946emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
4947 const OMPTargetParallelForSimdDirective &S,
4948 PrePostActionTy &Action) {
4949 Action.Enter(CGF);
4950 // Emit directive as a combined directive that consists of two implicit
4951 // directives: 'parallel' with 'for' directive.
4952 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4953 Action.Enter(CGF);
4954 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4955 emitDispatchForLoopBounds);
4956 };
4957 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4958 emitEmptyBoundParameters);
4959}
4960
4961void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
4962 CodeGenModule &CGM, StringRef ParentName,
4963 const OMPTargetParallelForSimdDirective &S) {
4964 // Emit SPMD target parallel for region as a standalone region.
4965 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4966 emitTargetParallelForSimdRegion(CGF, S, Action);
4967 };
4968 llvm::Function *Fn;
4969 llvm::Constant *Addr;
4970 // Emit target region as a standalone region.
4971 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4972 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4973 assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed."
) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4973, __PRETTY_FUNCTION__))
;
4974}
4975
4976void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
4977 const OMPTargetParallelForSimdDirective &S) {
4978 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4979 emitTargetParallelForSimdRegion(CGF, S, Action);
4980 };
4981 emitCommonOMPTargetDirective(*this, S, CodeGen);
4982}
4983
4984/// Emit a helper variable and return corresponding lvalue.
4985static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4986 const ImplicitParamDecl *PVD,
4987 CodeGenFunction::OMPPrivateScope &Privates) {
4988 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4989 Privates.addPrivate(VDecl,
4990 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4991}
4992
4993void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
4994 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()))((isOpenMPTaskLoopDirective(S.getDirectiveKind())) ? static_cast
<void> (0) : __assert_fail ("isOpenMPTaskLoopDirective(S.getDirectiveKind())"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/CodeGen/CGStmtOpenMP.cpp"
, 4994, __PRETTY_FUNCTION__))
;
4995 // Emit outlined function for task construct.
4996 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4997 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4998 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4999 const Expr *IfCond = nullptr;
5000 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5001 if (C->getNameModifier() == OMPD_unknown ||
5002 C->getNameModifier() == OMPD_taskloop) {
5003 IfCond = C->getCondition();
5004 break;
5005 }
5006 }
5007
5008 OMPTaskDataTy Data;
5009 // Check if taskloop must be emitted without taskgroup.
5010 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
5011 // TODO: Check if we should emit tied or untied task.
5012 Data.Tied = true;
5013 // Set scheduling for taskloop
5014 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
5015 // grainsize clause
5016 Data.Schedule.setInt(/*IntVal=*/false);
5017 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
5018 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
5019 // num_tasks clause
5020 Data.Schedule.setInt(/*IntVal=*/true);
5021 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
5022 }
5023
5024 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
5025 // if (PreCond) {
5026 // for (IV in 0..LastIteration) BODY;
5027 // <Final counter/linear vars updates>;
5028 // }
5029 //
5030
5031 // Emit: if (PreCond) - begin.
5032 // If the condition constant folds and can be elided, avoid emitting the
5033 // whole loop.
5034 bool CondConstant;
5035 llvm::BasicBlock *ContBlock = nullptr;
5036 OMPLoopScope PreInitScope(CGF, S);
5037 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5038 if (!CondConstant)
5039 return;
5040 } else {
5041 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
5042 ContBlock = CGF.createBasicBlock("taskloop.if.end");
5043 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
5044 CGF.getProfileCount(&S));
5045 CGF.EmitBlock(ThenBlock);
5046 CGF.incrementProfileCounter(&S);
5047 }
5048
5049 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
5050 CGF.EmitOMPSimdInit(S);
5051 (void)CGF.EmitOMPLinearClauseInit(S);
5052 }
5053
5054 OMPPrivateScope LoopScope(CGF);
5055 // Emit helper vars inits.
5056 enum { LowerBound = 5, UpperBound, Stride, LastIter };
5057 auto *I = CS->getCapturedDecl()->param_begin();
5058 auto *LBP = std::next(I, LowerBound);
5059 auto *UBP = std::next(I, UpperBound);
5060 auto *STP = std::next(I, Stride);
5061 auto *LIP = std::next(I, LastIter);
5062 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
5063 LoopScope);
5064 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
5065 LoopScope);
5066 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
5067 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
5068 LoopScope);
5069 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
5070 CGF.EmitOMPLinearClause(S, LoopScope);
5071 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
5072 (void)LoopScope.Privatize();
5073 // Emit the loop iteration variable.
5074 const Expr *IVExpr = S.getIterationVariable();
5075 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
5076 CGF.EmitVarDecl(*IVDecl);
5077 CGF.EmitIgnoredExpr(S.getInit());
5078
5079 // Emit the iterations count variable.
5080 // If it is not a variable, Sema decided to calculate iterations count on
5081 // each iteration (e.g., it is foldable into a constant).
5082 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5083 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5084 // Emit calculation of the iterations count.
5085 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
5086 }
5087
5088 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
5089 S.getInc(),
5090 [&S](CodeGenFunction &CGF) {
5091 CGF.EmitOMPLoopBody(S, JumpDest());
5092 CGF.EmitStopPoint(&S);
5093 },
5094 [](CodeGenFunction &) {});
5095 // Emit: if (PreCond) - end.
5096 if (ContBlock) {
5097 CGF.EmitBranch(ContBlock);
5098 CGF.EmitBlock(ContBlock, true);
5099 }
5100 // Emit final copy of the lastprivate variables if IsLastIter != 0.
5101 if (HasLastprivateClause) {
5102 CGF.EmitOMPLastprivateClauseFinal(
5103 S, isOpenMPSimdDirective(S.getDirectiveKind()),
5104 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
5105 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5106 (*LIP)->getType(), S.getBeginLoc())));
5107 }
5108 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
5109 return CGF.Builder.CreateIsNotNull(
5110 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5111 (*LIP)->getType(), S.getBeginLoc()));
5112 });
5113 };
5114 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5115 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5116 const OMPTaskDataTy &Data) {
5117 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
5118 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
5119 OMPLoopScope PreInitScope(CGF, S);
5120 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
5121 OutlinedFn, SharedsTy,
5122 CapturedStruct, IfCond, Data);
5123 };
5124 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5125 CodeGen);
5126 };
5127 if (Data.Nogroup) {
5128 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5129 } else {
5130 CGM.getOpenMPRuntime().emitTaskgroupRegion(
5131 *this,
5132 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5133 PrePostActionTy &Action) {
5134 Action.Enter(CGF);
5135 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5136 Data);
5137 },
5138 S.getBeginLoc());
5139 }
5140}
5141
5142void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
5143 EmitOMPTaskLoopBasedDirective(S);
5144}
5145
5146void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
5147 const OMPTaskLoopSimdDirective &S) {
5148 EmitOMPTaskLoopBasedDirective(S);
5149}
5150
5151void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
5152 const OMPMasterTaskLoopDirective &S) {
5153 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5154 Action.Enter(CGF);
5155 EmitOMPTaskLoopBasedDirective(S);
5156 };
5157 OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false);
5158 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5159}
5160
5161void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
5162 const OMPMasterTaskLoopSimdDirective &S) {
5163 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5164 Action.Enter(CGF);
5165 EmitOMPTaskLoopBasedDirective(S);
5166 };
5167 OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false);
5168 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5169}
5170
5171void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
5172 const OMPParallelMasterTaskLoopDirective &S) {
5173 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5174 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5175 PrePostActionTy &Action) {
5176 Action.Enter(CGF);
5177 CGF.EmitOMPTaskLoopBasedDirective(S);
5178 };
5179 OMPLexicalScope Scope(CGF, S, llvm::None, /*EmitPreInitStmt=*/false);
5180 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5181 S.getBeginLoc());
5182 };
5183 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
5184 emitEmptyBoundParameters);
5185}
5186
5187void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
5188 const OMPParallelMasterTaskLoopSimdDirective &S) {
5189 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5190 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5191 PrePostActionTy &Action) {
5192 Action.Enter(CGF);
5193 CGF.EmitOMPTaskLoopBasedDirective(S);
5194 };
5195 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
5196 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5197 S.getBeginLoc());
5198 };
5199 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
5200 emitEmptyBoundParameters);
5201}
5202
5203// Generate the instructions for '#pragma omp target update' directive.
5204void CodeGenFunction::EmitOMPTargetUpdateDirective(
5205 const OMPTargetUpdateDirective &S) {
5206 // If we don't have target devices, don't bother emitting the data mapping
5207 // code.
5208 if (CGM.getLangOpts().OMPTargetTriples.empty())
5209 return;
5210
5211 // Check if we have any if clause associated with the directive.
5212 const Expr *IfCond = nullptr;
5213 if (const auto *C = S.getSingleClause<OMPIfClause>())
5214 IfCond = C->getCondition();
5215
5216 // Check if we have any device clause associated with the directive.
5217 const Expr *Device = nullptr;
5218 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5219 Device = C->getDevice();
5220
5221 OMPLexicalScope Scope(*this, S, OMPD_task);
5222 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5223}
5224
5225void CodeGenFunction::EmitSimpleOMPExecutableDirective(
5226 const OMPExecutableDirective &D) {
5227 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5228 return;
5229 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5230 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1
Assuming the condition is false
2
Taking false branch
5231 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5232 } else {
5233 OMPPrivateScope LoopGlobals(CGF);
5234 if (const auto *LD
3.1
'LD' is non-null
= dyn_cast<OMPLoopDirective>(&D)) {
3
Assuming the object is a 'OMPLoopDirective'
4
Taking true branch
5235 for (const Expr *E : LD->counters()) {
5
Assuming '__begin3' is not equal to '__end3'
5236 const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
6
'E' is a 'DeclRefExpr'
7
Assuming the object is not a 'VarDecl'
8
'VD' initialized to a null pointer value
5237 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
9
Called C++ object pointer is null
5238 LValue GlobLVal = CGF.EmitLValue(E);
5239 LoopGlobals.addPrivate(
5240 VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5241 }
5242 if (isa<OMPCapturedExprDecl>(VD)) {
5243 // Emit only those that were not explicitly referenced in clauses.
5244 if (!CGF.LocalDeclMap.count(VD))
5245 CGF.EmitVarDecl(*VD);
5246 }
5247 }
5248 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5249 if (!C->getNumForLoops())
5250 continue;
5251 for (unsigned I = LD->getCollapsedNumber(),
5252 E = C->getLoopNumIterations().size();
5253 I < E; ++I) {
5254 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5255 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5256 // Emit only those that were not explicitly referenced in clauses.
5257 if (!CGF.LocalDeclMap.count(VD))
5258 CGF.EmitVarDecl(*VD);
5259 }
5260 }
5261 }
5262 }
5263 LoopGlobals.Privatize();
5264 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
5265 }
5266 };
5267 OMPSimdLexicalScope Scope(*this, D);
5268 CGM.getOpenMPRuntime().emitInlinedDirective(
5269 *this,
5270 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5271 : D.getDirectiveKind(),
5272 CodeGen);
5273}