File: | clang/lib/CodeGen/CGStmtOpenMP.cpp |
Warning: | line 5475, column 16 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This contains code to emit OpenMP nodes as LLVM code. | |||
10 | // | |||
11 | //===----------------------------------------------------------------------===// | |||
12 | ||||
13 | #include "CGCleanup.h" | |||
14 | #include "CGOpenMPRuntime.h" | |||
15 | #include "CodeGenFunction.h" | |||
16 | #include "CodeGenModule.h" | |||
17 | #include "TargetInfo.h" | |||
18 | #include "clang/AST/ASTContext.h" | |||
19 | #include "clang/AST/Attr.h" | |||
20 | #include "clang/AST/DeclOpenMP.h" | |||
21 | #include "clang/AST/Stmt.h" | |||
22 | #include "clang/AST/StmtOpenMP.h" | |||
23 | #include "clang/Basic/PrettyStackTrace.h" | |||
24 | #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" | |||
25 | using namespace clang; | |||
26 | using namespace CodeGen; | |||
27 | using namespace llvm::omp; | |||
28 | ||||
29 | namespace { | |||
30 | /// Lexical scope for OpenMP executable constructs, that handles correct codegen | |||
31 | /// for captured expressions. | |||
32 | class OMPLexicalScope : public CodeGenFunction::LexicalScope { | |||
33 | void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { | |||
34 | for (const auto *C : S.clauses()) { | |||
35 | if (const auto *CPI = OMPClauseWithPreInit::get(C)) { | |||
36 | if (const auto *PreInit = | |||
37 | cast_or_null<DeclStmt>(CPI->getPreInitStmt())) { | |||
38 | for (const auto *I : PreInit->decls()) { | |||
39 | if (!I->hasAttr<OMPCaptureNoInitAttr>()) { | |||
40 | CGF.EmitVarDecl(cast<VarDecl>(*I)); | |||
41 | } else { | |||
42 | CodeGenFunction::AutoVarEmission Emission = | |||
43 | CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); | |||
44 | CGF.EmitAutoVarCleanups(Emission); | |||
45 | } | |||
46 | } | |||
47 | } | |||
48 | } | |||
49 | } | |||
50 | } | |||
51 | CodeGenFunction::OMPPrivateScope InlinedShareds; | |||
52 | ||||
53 | static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { | |||
54 | return CGF.LambdaCaptureFields.lookup(VD) || | |||
55 | (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || | |||
56 | (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl)); | |||
57 | } | |||
58 | ||||
59 | public: | |||
60 | OMPLexicalScope( | |||
61 | CodeGenFunction &CGF, const OMPExecutableDirective &S, | |||
62 | const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None, | |||
63 | const bool EmitPreInitStmt = true) | |||
64 | : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), | |||
65 | InlinedShareds(CGF) { | |||
66 | if (EmitPreInitStmt) | |||
67 | emitPreInitStmt(CGF, S); | |||
68 | if (!CapturedRegion.hasValue()) | |||
69 | return; | |||
70 | assert(S.hasAssociatedStmt() &&((S.hasAssociatedStmt() && "Expected associated statement for inlined directive." ) ? static_cast<void> (0) : __assert_fail ("S.hasAssociatedStmt() && \"Expected associated statement for inlined directive.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 71, __PRETTY_FUNCTION__)) | |||
71 | "Expected associated statement for inlined directive.")((S.hasAssociatedStmt() && "Expected associated statement for inlined directive." ) ? static_cast<void> (0) : __assert_fail ("S.hasAssociatedStmt() && \"Expected associated statement for inlined directive.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 71, __PRETTY_FUNCTION__)); | |||
72 | const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion); | |||
73 | for (const auto &C : CS->captures()) { | |||
74 | if (C.capturesVariable() || C.capturesVariableByCopy()) { | |||
75 | auto *VD = C.getCapturedVar(); | |||
76 | assert(VD == VD->getCanonicalDecl() &&((VD == VD->getCanonicalDecl() && "Canonical decl must be captured." ) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 77, __PRETTY_FUNCTION__)) | |||
77 | "Canonical decl must be captured.")((VD == VD->getCanonicalDecl() && "Canonical decl must be captured." ) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 77, __PRETTY_FUNCTION__)); | |||
78 | DeclRefExpr DRE( | |||
79 | CGF.getContext(), const_cast<VarDecl *>(VD), | |||
80 | isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && | |||
81 | InlinedShareds.isGlobalVarCaptured(VD)), | |||
82 | VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); | |||
83 | InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { | |||
84 | return CGF.EmitLValue(&DRE).getAddress(CGF); | |||
85 | }); | |||
86 | } | |||
87 | } | |||
88 | (void)InlinedShareds.Privatize(); | |||
89 | } | |||
90 | }; | |||
91 | ||||
92 | /// Lexical scope for OpenMP parallel construct, that handles correct codegen | |||
93 | /// for captured expressions. | |||
94 | class OMPParallelScope final : public OMPLexicalScope { | |||
95 | bool EmitPreInitStmt(const OMPExecutableDirective &S) { | |||
96 | OpenMPDirectiveKind Kind = S.getDirectiveKind(); | |||
97 | return !(isOpenMPTargetExecutionDirective(Kind) || | |||
98 | isOpenMPLoopBoundSharingDirective(Kind)) && | |||
99 | isOpenMPParallelDirective(Kind); | |||
100 | } | |||
101 | ||||
102 | public: | |||
103 | OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) | |||
104 | : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, | |||
105 | EmitPreInitStmt(S)) {} | |||
106 | }; | |||
107 | ||||
108 | /// Lexical scope for OpenMP teams construct, that handles correct codegen | |||
109 | /// for captured expressions. | |||
110 | class OMPTeamsScope final : public OMPLexicalScope { | |||
111 | bool EmitPreInitStmt(const OMPExecutableDirective &S) { | |||
112 | OpenMPDirectiveKind Kind = S.getDirectiveKind(); | |||
113 | return !isOpenMPTargetExecutionDirective(Kind) && | |||
114 | isOpenMPTeamsDirective(Kind); | |||
115 | } | |||
116 | ||||
117 | public: | |||
118 | OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) | |||
119 | : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, | |||
120 | EmitPreInitStmt(S)) {} | |||
121 | }; | |||
122 | ||||
123 | /// Private scope for OpenMP loop-based directives, that supports capturing | |||
124 | /// of used expression from loop statement. | |||
125 | class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { | |||
126 | void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { | |||
127 | CodeGenFunction::OMPMapVars PreCondVars; | |||
128 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; | |||
129 | for (const auto *E : S.counters()) { | |||
130 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
131 | EmittedAsPrivate.insert(VD->getCanonicalDecl()); | |||
132 | (void)PreCondVars.setVarAddr( | |||
133 | CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); | |||
134 | } | |||
135 | // Mark private vars as undefs. | |||
136 | for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { | |||
137 | for (const Expr *IRef : C->varlists()) { | |||
138 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); | |||
139 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
140 | (void)PreCondVars.setVarAddr( | |||
141 | CGF, OrigVD, | |||
142 | Address(llvm::UndefValue::get( | |||
143 | CGF.ConvertTypeForMem(CGF.getContext().getPointerType( | |||
144 | OrigVD->getType().getNonReferenceType()))), | |||
145 | CGF.getContext().getDeclAlign(OrigVD))); | |||
146 | } | |||
147 | } | |||
148 | } | |||
149 | (void)PreCondVars.apply(CGF); | |||
150 | // Emit init, __range and __end variables for C++ range loops. | |||
151 | const Stmt *Body = | |||
152 | S.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(); | |||
153 | for (unsigned Cnt = 0; Cnt < S.getCollapsedNumber(); ++Cnt) { | |||
154 | Body = OMPLoopDirective::tryToFindNextInnerLoop( | |||
155 | Body, /*TryImperfectlyNestedLoops=*/true); | |||
156 | if (auto *For = dyn_cast<ForStmt>(Body)) { | |||
157 | Body = For->getBody(); | |||
158 | } else { | |||
159 | assert(isa<CXXForRangeStmt>(Body) &&((isa<CXXForRangeStmt>(Body) && "Expected canonical for loop or range-based for loop." ) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(Body) && \"Expected canonical for loop or range-based for loop.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 160, __PRETTY_FUNCTION__)) | |||
160 | "Expected canonical for loop or range-based for loop.")((isa<CXXForRangeStmt>(Body) && "Expected canonical for loop or range-based for loop." ) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(Body) && \"Expected canonical for loop or range-based for loop.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 160, __PRETTY_FUNCTION__)); | |||
161 | auto *CXXFor = cast<CXXForRangeStmt>(Body); | |||
162 | if (const Stmt *Init = CXXFor->getInit()) | |||
163 | CGF.EmitStmt(Init); | |||
164 | CGF.EmitStmt(CXXFor->getRangeStmt()); | |||
165 | CGF.EmitStmt(CXXFor->getEndStmt()); | |||
166 | Body = CXXFor->getBody(); | |||
167 | } | |||
168 | } | |||
169 | if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) { | |||
170 | for (const auto *I : PreInits->decls()) | |||
171 | CGF.EmitVarDecl(cast<VarDecl>(*I)); | |||
172 | } | |||
173 | PreCondVars.restore(CGF); | |||
174 | } | |||
175 | ||||
176 | public: | |||
177 | OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) | |||
178 | : CodeGenFunction::RunCleanupsScope(CGF) { | |||
179 | emitPreInitStmt(CGF, S); | |||
180 | } | |||
181 | }; | |||
182 | ||||
183 | class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { | |||
184 | CodeGenFunction::OMPPrivateScope InlinedShareds; | |||
185 | ||||
186 | static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { | |||
187 | return CGF.LambdaCaptureFields.lookup(VD) || | |||
188 | (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || | |||
189 | (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && | |||
190 | cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); | |||
191 | } | |||
192 | ||||
193 | public: | |||
194 | OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) | |||
195 | : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), | |||
196 | InlinedShareds(CGF) { | |||
197 | for (const auto *C : S.clauses()) { | |||
198 | if (const auto *CPI = OMPClauseWithPreInit::get(C)) { | |||
199 | if (const auto *PreInit = | |||
200 | cast_or_null<DeclStmt>(CPI->getPreInitStmt())) { | |||
201 | for (const auto *I : PreInit->decls()) { | |||
202 | if (!I->hasAttr<OMPCaptureNoInitAttr>()) { | |||
203 | CGF.EmitVarDecl(cast<VarDecl>(*I)); | |||
204 | } else { | |||
205 | CodeGenFunction::AutoVarEmission Emission = | |||
206 | CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); | |||
207 | CGF.EmitAutoVarCleanups(Emission); | |||
208 | } | |||
209 | } | |||
210 | } | |||
211 | } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) { | |||
212 | for (const Expr *E : UDP->varlists()) { | |||
213 | const Decl *D = cast<DeclRefExpr>(E)->getDecl(); | |||
214 | if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D)) | |||
215 | CGF.EmitVarDecl(*OED); | |||
216 | } | |||
217 | } | |||
218 | } | |||
219 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) | |||
220 | CGF.EmitOMPPrivateClause(S, InlinedShareds); | |||
221 | if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) { | |||
222 | if (const Expr *E = TG->getReductionRef()) | |||
223 | CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); | |||
224 | } | |||
225 | const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt()); | |||
226 | while (CS) { | |||
227 | for (auto &C : CS->captures()) { | |||
228 | if (C.capturesVariable() || C.capturesVariableByCopy()) { | |||
229 | auto *VD = C.getCapturedVar(); | |||
230 | assert(VD == VD->getCanonicalDecl() &&((VD == VD->getCanonicalDecl() && "Canonical decl must be captured." ) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 231, __PRETTY_FUNCTION__)) | |||
231 | "Canonical decl must be captured.")((VD == VD->getCanonicalDecl() && "Canonical decl must be captured." ) ? static_cast<void> (0) : __assert_fail ("VD == VD->getCanonicalDecl() && \"Canonical decl must be captured.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 231, __PRETTY_FUNCTION__)); | |||
232 | DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), | |||
233 | isCapturedVar(CGF, VD) || | |||
234 | (CGF.CapturedStmtInfo && | |||
235 | InlinedShareds.isGlobalVarCaptured(VD)), | |||
236 | VD->getType().getNonReferenceType(), VK_LValue, | |||
237 | C.getLocation()); | |||
238 | InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { | |||
239 | return CGF.EmitLValue(&DRE).getAddress(CGF); | |||
240 | }); | |||
241 | } | |||
242 | } | |||
243 | CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()); | |||
244 | } | |||
245 | (void)InlinedShareds.Privatize(); | |||
246 | } | |||
247 | }; | |||
248 | ||||
249 | } // namespace | |||
250 | ||||
251 | static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, | |||
252 | const OMPExecutableDirective &S, | |||
253 | const RegionCodeGenTy &CodeGen); | |||
254 | ||||
255 | LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { | |||
256 | if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) { | |||
257 | if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) { | |||
258 | OrigVD = OrigVD->getCanonicalDecl(); | |||
259 | bool IsCaptured = | |||
260 | LambdaCaptureFields.lookup(OrigVD) || | |||
261 | (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || | |||
262 | (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)); | |||
263 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured, | |||
264 | OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); | |||
265 | return EmitLValue(&DRE); | |||
266 | } | |||
267 | } | |||
268 | return EmitLValue(E); | |||
269 | } | |||
270 | ||||
271 | llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) { | |||
272 | ASTContext &C = getContext(); | |||
273 | llvm::Value *Size = nullptr; | |||
274 | auto SizeInChars = C.getTypeSizeInChars(Ty); | |||
275 | if (SizeInChars.isZero()) { | |||
276 | // getTypeSizeInChars() returns 0 for a VLA. | |||
277 | while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) { | |||
278 | VlaSizePair VlaSize = getVLASize(VAT); | |||
279 | Ty = VlaSize.Type; | |||
280 | Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) | |||
281 | : VlaSize.NumElts; | |||
282 | } | |||
283 | SizeInChars = C.getTypeSizeInChars(Ty); | |||
284 | if (SizeInChars.isZero()) | |||
285 | return llvm::ConstantInt::get(SizeTy, /*V=*/0); | |||
286 | return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); | |||
287 | } | |||
288 | return CGM.getSize(SizeInChars); | |||
289 | } | |||
290 | ||||
291 | void CodeGenFunction::GenerateOpenMPCapturedVars( | |||
292 | const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) { | |||
293 | const RecordDecl *RD = S.getCapturedRecordDecl(); | |||
294 | auto CurField = RD->field_begin(); | |||
295 | auto CurCap = S.captures().begin(); | |||
296 | for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(), | |||
297 | E = S.capture_init_end(); | |||
298 | I != E; ++I, ++CurField, ++CurCap) { | |||
299 | if (CurField->hasCapturedVLAType()) { | |||
300 | const VariableArrayType *VAT = CurField->getCapturedVLAType(); | |||
301 | llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()]; | |||
302 | CapturedVars.push_back(Val); | |||
303 | } else if (CurCap->capturesThis()) { | |||
304 | CapturedVars.push_back(CXXThisValue); | |||
305 | } else if (CurCap->capturesVariableByCopy()) { | |||
306 | llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation()); | |||
307 | ||||
308 | // If the field is not a pointer, we need to save the actual value | |||
309 | // and load it as a void pointer. | |||
310 | if (!CurField->getType()->isAnyPointerType()) { | |||
311 | ASTContext &Ctx = getContext(); | |||
312 | Address DstAddr = CreateMemTemp( | |||
313 | Ctx.getUIntPtrType(), | |||
314 | Twine(CurCap->getCapturedVar()->getName(), ".casted")); | |||
315 | LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); | |||
316 | ||||
317 | llvm::Value *SrcAddrVal = EmitScalarConversion( | |||
318 | DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), | |||
319 | Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); | |||
320 | LValue SrcLV = | |||
321 | MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); | |||
322 | ||||
323 | // Store the value using the source type pointer. | |||
324 | EmitStoreThroughLValue(RValue::get(CV), SrcLV); | |||
325 | ||||
326 | // Load the value using the destination type pointer. | |||
327 | CV = EmitLoadOfScalar(DstLV, CurCap->getLocation()); | |||
328 | } | |||
329 | CapturedVars.push_back(CV); | |||
330 | } else { | |||
331 | assert(CurCap->capturesVariable() && "Expected capture by reference.")((CurCap->capturesVariable() && "Expected capture by reference." ) ? static_cast<void> (0) : __assert_fail ("CurCap->capturesVariable() && \"Expected capture by reference.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 331, __PRETTY_FUNCTION__)); | |||
332 | CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); | |||
333 | } | |||
334 | } | |||
335 | } | |||
336 | ||||
337 | static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, | |||
338 | QualType DstType, StringRef Name, | |||
339 | LValue AddrLV) { | |||
340 | ASTContext &Ctx = CGF.getContext(); | |||
341 | ||||
342 | llvm::Value *CastedPtr = CGF.EmitScalarConversion( | |||
343 | AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), | |||
344 | Ctx.getPointerType(DstType), Loc); | |||
345 | Address TmpAddr = | |||
346 | CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) | |||
347 | .getAddress(CGF); | |||
348 | return TmpAddr; | |||
349 | } | |||
350 | ||||
351 | static QualType getCanonicalParamType(ASTContext &C, QualType T) { | |||
352 | if (T->isLValueReferenceType()) | |||
353 | return C.getLValueReferenceType( | |||
354 | getCanonicalParamType(C, T.getNonReferenceType()), | |||
355 | /*SpelledAsLValue=*/false); | |||
356 | if (T->isPointerType()) | |||
357 | return C.getPointerType(getCanonicalParamType(C, T->getPointeeType())); | |||
358 | if (const ArrayType *A = T->getAsArrayTypeUnsafe()) { | |||
359 | if (const auto *VLA = dyn_cast<VariableArrayType>(A)) | |||
360 | return getCanonicalParamType(C, VLA->getElementType()); | |||
361 | if (!A->isVariablyModifiedType()) | |||
362 | return C.getCanonicalType(T); | |||
363 | } | |||
364 | return C.getCanonicalParamType(T); | |||
365 | } | |||
366 | ||||
367 | namespace { | |||
368 | /// Contains required data for proper outlined function codegen. | |||
369 | struct FunctionOptions { | |||
370 | /// Captured statement for which the function is generated. | |||
371 | const CapturedStmt *S = nullptr; | |||
372 | /// true if cast to/from UIntPtr is required for variables captured by | |||
373 | /// value. | |||
374 | const bool UIntPtrCastRequired = true; | |||
375 | /// true if only casted arguments must be registered as local args or VLA | |||
376 | /// sizes. | |||
377 | const bool RegisterCastedArgsOnly = false; | |||
378 | /// Name of the generated function. | |||
379 | const StringRef FunctionName; | |||
380 | explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired, | |||
381 | bool RegisterCastedArgsOnly, | |||
382 | StringRef FunctionName) | |||
383 | : S(S), UIntPtrCastRequired(UIntPtrCastRequired), | |||
384 | RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly), | |||
385 | FunctionName(FunctionName) {} | |||
386 | }; | |||
387 | } | |||
388 | ||||
389 | static llvm::Function *emitOutlinedFunctionPrologue( | |||
390 | CodeGenFunction &CGF, FunctionArgList &Args, | |||
391 | llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> | |||
392 | &LocalAddrs, | |||
393 | llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> | |||
394 | &VLASizes, | |||
395 | llvm::Value *&CXXThisValue, const FunctionOptions &FO) { | |||
396 | const CapturedDecl *CD = FO.S->getCapturedDecl(); | |||
397 | const RecordDecl *RD = FO.S->getCapturedRecordDecl(); | |||
398 | assert(CD->hasBody() && "missing CapturedDecl body")((CD->hasBody() && "missing CapturedDecl body") ? static_cast <void> (0) : __assert_fail ("CD->hasBody() && \"missing CapturedDecl body\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 398, __PRETTY_FUNCTION__)); | |||
399 | ||||
400 | CXXThisValue = nullptr; | |||
401 | // Build the argument list. | |||
402 | CodeGenModule &CGM = CGF.CGM; | |||
403 | ASTContext &Ctx = CGM.getContext(); | |||
404 | FunctionArgList TargetArgs; | |||
405 | Args.append(CD->param_begin(), | |||
406 | std::next(CD->param_begin(), CD->getContextParamPosition())); | |||
407 | TargetArgs.append( | |||
408 | CD->param_begin(), | |||
409 | std::next(CD->param_begin(), CD->getContextParamPosition())); | |||
410 | auto I = FO.S->captures().begin(); | |||
411 | FunctionDecl *DebugFunctionDecl = nullptr; | |||
412 | if (!FO.UIntPtrCastRequired) { | |||
413 | FunctionProtoType::ExtProtoInfo EPI; | |||
414 | QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI); | |||
415 | DebugFunctionDecl = FunctionDecl::Create( | |||
416 | Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(), | |||
417 | SourceLocation(), DeclarationName(), FunctionTy, | |||
418 | Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static, | |||
419 | /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false); | |||
420 | } | |||
421 | for (const FieldDecl *FD : RD->fields()) { | |||
422 | QualType ArgType = FD->getType(); | |||
423 | IdentifierInfo *II = nullptr; | |||
424 | VarDecl *CapVar = nullptr; | |||
425 | ||||
426 | // If this is a capture by copy and the type is not a pointer, the outlined | |||
427 | // function argument type should be uintptr and the value properly casted to | |||
428 | // uintptr. This is necessary given that the runtime library is only able to | |||
429 | // deal with pointers. We can pass in the same way the VLA type sizes to the | |||
430 | // outlined function. | |||
431 | if (FO.UIntPtrCastRequired && | |||
432 | ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || | |||
433 | I->capturesVariableArrayType())) | |||
434 | ArgType = Ctx.getUIntPtrType(); | |||
435 | ||||
436 | if (I->capturesVariable() || I->capturesVariableByCopy()) { | |||
437 | CapVar = I->getCapturedVar(); | |||
438 | II = CapVar->getIdentifier(); | |||
439 | } else if (I->capturesThis()) { | |||
440 | II = &Ctx.Idents.get("this"); | |||
441 | } else { | |||
442 | assert(I->capturesVariableArrayType())((I->capturesVariableArrayType()) ? static_cast<void> (0) : __assert_fail ("I->capturesVariableArrayType()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 442, __PRETTY_FUNCTION__)); | |||
443 | II = &Ctx.Idents.get("vla"); | |||
444 | } | |||
445 | if (ArgType->isVariablyModifiedType()) | |||
446 | ArgType = getCanonicalParamType(Ctx, ArgType); | |||
447 | VarDecl *Arg; | |||
448 | if (DebugFunctionDecl && (CapVar || I->capturesThis())) { | |||
449 | Arg = ParmVarDecl::Create( | |||
450 | Ctx, DebugFunctionDecl, | |||
451 | CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(), | |||
452 | CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType, | |||
453 | /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr); | |||
454 | } else { | |||
455 | Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), | |||
456 | II, ArgType, ImplicitParamDecl::Other); | |||
457 | } | |||
458 | Args.emplace_back(Arg); | |||
459 | // Do not cast arguments if we emit function with non-original types. | |||
460 | TargetArgs.emplace_back( | |||
461 | FO.UIntPtrCastRequired | |||
462 | ? Arg | |||
463 | : CGM.getOpenMPRuntime().translateParameter(FD, Arg)); | |||
464 | ++I; | |||
465 | } | |||
466 | Args.append( | |||
467 | std::next(CD->param_begin(), CD->getContextParamPosition() + 1), | |||
468 | CD->param_end()); | |||
469 | TargetArgs.append( | |||
470 | std::next(CD->param_begin(), CD->getContextParamPosition() + 1), | |||
471 | CD->param_end()); | |||
472 | ||||
473 | // Create the function declaration. | |||
474 | const CGFunctionInfo &FuncInfo = | |||
475 | CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs); | |||
476 | llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo); | |||
477 | ||||
478 | auto *F = | |||
479 | llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage, | |||
480 | FO.FunctionName, &CGM.getModule()); | |||
481 | CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); | |||
482 | if (CD->isNothrow()) | |||
483 | F->setDoesNotThrow(); | |||
484 | F->setDoesNotRecurse(); | |||
485 | ||||
486 | // Generate the function. | |||
487 | CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, | |||
488 | FO.S->getBeginLoc(), CD->getBody()->getBeginLoc()); | |||
489 | unsigned Cnt = CD->getContextParamPosition(); | |||
490 | I = FO.S->captures().begin(); | |||
491 | for (const FieldDecl *FD : RD->fields()) { | |||
492 | // Do not map arguments if we emit function with non-original types. | |||
493 | Address LocalAddr(Address::invalid()); | |||
494 | if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) { | |||
495 | LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], | |||
496 | TargetArgs[Cnt]); | |||
497 | } else { | |||
498 | LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); | |||
499 | } | |||
500 | // If we are capturing a pointer by copy we don't need to do anything, just | |||
501 | // use the value that we get from the arguments. | |||
502 | if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) { | |||
503 | const VarDecl *CurVD = I->getCapturedVar(); | |||
504 | if (!FO.RegisterCastedArgsOnly) | |||
505 | LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}}); | |||
506 | ++Cnt; | |||
507 | ++I; | |||
508 | continue; | |||
509 | } | |||
510 | ||||
511 | LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), | |||
512 | AlignmentSource::Decl); | |||
513 | if (FD->hasCapturedVLAType()) { | |||
514 | if (FO.UIntPtrCastRequired) { | |||
515 | ArgLVal = CGF.MakeAddrLValue( | |||
516 | castValueFromUintptr(CGF, I->getLocation(), FD->getType(), | |||
517 | Args[Cnt]->getName(), ArgLVal), | |||
518 | FD->getType(), AlignmentSource::Decl); | |||
519 | } | |||
520 | llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); | |||
521 | const VariableArrayType *VAT = FD->getCapturedVLAType(); | |||
522 | VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg); | |||
523 | } else if (I->capturesVariable()) { | |||
524 | const VarDecl *Var = I->getCapturedVar(); | |||
525 | QualType VarTy = Var->getType(); | |||
526 | Address ArgAddr = ArgLVal.getAddress(CGF); | |||
527 | if (ArgLVal.getType()->isLValueReferenceType()) { | |||
528 | ArgAddr = CGF.EmitLoadOfReference(ArgLVal); | |||
529 | } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { | |||
530 | assert(ArgLVal.getType()->isPointerType())((ArgLVal.getType()->isPointerType()) ? static_cast<void > (0) : __assert_fail ("ArgLVal.getType()->isPointerType()" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 530, __PRETTY_FUNCTION__)); | |||
531 | ArgAddr = CGF.EmitLoadOfPointer( | |||
532 | ArgAddr, ArgLVal.getType()->castAs<PointerType>()); | |||
533 | } | |||
534 | if (!FO.RegisterCastedArgsOnly) { | |||
535 | LocalAddrs.insert( | |||
536 | {Args[Cnt], | |||
537 | {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}}); | |||
538 | } | |||
539 | } else if (I->capturesVariableByCopy()) { | |||
540 | assert(!FD->getType()->isAnyPointerType() &&((!FD->getType()->isAnyPointerType() && "Not expecting a captured pointer." ) ? static_cast<void> (0) : __assert_fail ("!FD->getType()->isAnyPointerType() && \"Not expecting a captured pointer.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 541, __PRETTY_FUNCTION__)) | |||
541 | "Not expecting a captured pointer.")((!FD->getType()->isAnyPointerType() && "Not expecting a captured pointer." ) ? static_cast<void> (0) : __assert_fail ("!FD->getType()->isAnyPointerType() && \"Not expecting a captured pointer.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 541, __PRETTY_FUNCTION__)); | |||
542 | const VarDecl *Var = I->getCapturedVar(); | |||
543 | LocalAddrs.insert({Args[Cnt], | |||
544 | {Var, FO.UIntPtrCastRequired | |||
545 | ? castValueFromUintptr( | |||
546 | CGF, I->getLocation(), FD->getType(), | |||
547 | Args[Cnt]->getName(), ArgLVal) | |||
548 | : ArgLVal.getAddress(CGF)}}); | |||
549 | } else { | |||
550 | // If 'this' is captured, load it into CXXThisValue. | |||
551 | assert(I->capturesThis())((I->capturesThis()) ? static_cast<void> (0) : __assert_fail ("I->capturesThis()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 551, __PRETTY_FUNCTION__)); | |||
552 | CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); | |||
553 | LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); | |||
554 | } | |||
555 | ++Cnt; | |||
556 | ++I; | |||
557 | } | |||
558 | ||||
559 | return F; | |||
560 | } | |||
561 | ||||
562 | llvm::Function * | |||
563 | CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { | |||
564 | assert(((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function" ) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 566, __PRETTY_FUNCTION__)) | |||
565 | CapturedStmtInfo &&((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function" ) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 566, __PRETTY_FUNCTION__)) | |||
566 | "CapturedStmtInfo should be set when generating the captured function")((CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function" ) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo && \"CapturedStmtInfo should be set when generating the captured function\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 566, __PRETTY_FUNCTION__)); | |||
567 | const CapturedDecl *CD = S.getCapturedDecl(); | |||
568 | // Build the argument list. | |||
569 | bool NeedWrapperFunction = | |||
570 | getDebugInfo() && | |||
571 | CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo; | |||
572 | FunctionArgList Args; | |||
573 | llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs; | |||
574 | llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes; | |||
575 | SmallString<256> Buffer; | |||
576 | llvm::raw_svector_ostream Out(Buffer); | |||
577 | Out << CapturedStmtInfo->getHelperName(); | |||
578 | if (NeedWrapperFunction) | |||
579 | Out << "_debug__"; | |||
580 | FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false, | |||
581 | Out.str()); | |||
582 | llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, | |||
583 | VLASizes, CXXThisValue, FO); | |||
584 | CodeGenFunction::OMPPrivateScope LocalScope(*this); | |||
585 | for (const auto &LocalAddrPair : LocalAddrs) { | |||
586 | if (LocalAddrPair.second.first) { | |||
587 | LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() { | |||
588 | return LocalAddrPair.second.second; | |||
589 | }); | |||
590 | } | |||
591 | } | |||
592 | (void)LocalScope.Privatize(); | |||
593 | for (const auto &VLASizePair : VLASizes) | |||
594 | VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second; | |||
595 | PGO.assignRegionCounters(GlobalDecl(CD), F); | |||
596 | CapturedStmtInfo->EmitBody(*this, CD->getBody()); | |||
597 | (void)LocalScope.ForceCleanup(); | |||
598 | FinishFunction(CD->getBodyRBrace()); | |||
599 | if (!NeedWrapperFunction) | |||
600 | return F; | |||
601 | ||||
602 | FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true, | |||
603 | /*RegisterCastedArgsOnly=*/true, | |||
604 | CapturedStmtInfo->getHelperName()); | |||
605 | CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true); | |||
606 | WrapperCGF.CapturedStmtInfo = CapturedStmtInfo; | |||
607 | Args.clear(); | |||
608 | LocalAddrs.clear(); | |||
609 | VLASizes.clear(); | |||
610 | llvm::Function *WrapperF = | |||
611 | emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes, | |||
612 | WrapperCGF.CXXThisValue, WrapperFO); | |||
613 | llvm::SmallVector<llvm::Value *, 4> CallArgs; | |||
614 | for (const auto *Arg : Args) { | |||
615 | llvm::Value *CallArg; | |||
616 | auto I = LocalAddrs.find(Arg); | |||
617 | if (I != LocalAddrs.end()) { | |||
618 | LValue LV = WrapperCGF.MakeAddrLValue( | |||
619 | I->second.second, | |||
620 | I->second.first ? I->second.first->getType() : Arg->getType(), | |||
621 | AlignmentSource::Decl); | |||
622 | CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc()); | |||
623 | } else { | |||
624 | auto EI = VLASizes.find(Arg); | |||
625 | if (EI != VLASizes.end()) { | |||
626 | CallArg = EI->second.second; | |||
627 | } else { | |||
628 | LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg), | |||
629 | Arg->getType(), | |||
630 | AlignmentSource::Decl); | |||
631 | CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc()); | |||
632 | } | |||
633 | } | |||
634 | CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType())); | |||
635 | } | |||
636 | CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(), | |||
637 | F, CallArgs); | |||
638 | WrapperCGF.FinishFunction(); | |||
639 | return WrapperF; | |||
640 | } | |||
641 | ||||
642 | //===----------------------------------------------------------------------===// | |||
643 | // OpenMP Directive Emission | |||
644 | //===----------------------------------------------------------------------===// | |||
645 | void CodeGenFunction::EmitOMPAggregateAssign( | |||
646 | Address DestAddr, Address SrcAddr, QualType OriginalType, | |||
647 | const llvm::function_ref<void(Address, Address)> CopyGen) { | |||
648 | // Perform element-by-element initialization. | |||
649 | QualType ElementTy; | |||
650 | ||||
651 | // Drill down to the base element type on both arrays. | |||
652 | const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe(); | |||
653 | llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); | |||
654 | SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); | |||
655 | ||||
656 | llvm::Value *SrcBegin = SrcAddr.getPointer(); | |||
657 | llvm::Value *DestBegin = DestAddr.getPointer(); | |||
658 | // Cast from pointer to array type to pointer to single element. | |||
659 | llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements); | |||
660 | // The basic structure here is a while-do loop. | |||
661 | llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body"); | |||
662 | llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done"); | |||
663 | llvm::Value *IsEmpty = | |||
664 | Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); | |||
665 | Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); | |||
666 | ||||
667 | // Enter the loop body, making that address the current address. | |||
668 | llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); | |||
669 | EmitBlock(BodyBB); | |||
670 | ||||
671 | CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy); | |||
672 | ||||
673 | llvm::PHINode *SrcElementPHI = | |||
674 | Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); | |||
675 | SrcElementPHI->addIncoming(SrcBegin, EntryBB); | |||
676 | Address SrcElementCurrent = | |||
677 | Address(SrcElementPHI, | |||
678 | SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |||
679 | ||||
680 | llvm::PHINode *DestElementPHI = | |||
681 | Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); | |||
682 | DestElementPHI->addIncoming(DestBegin, EntryBB); | |||
683 | Address DestElementCurrent = | |||
684 | Address(DestElementPHI, | |||
685 | DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |||
686 | ||||
687 | // Emit copy. | |||
688 | CopyGen(DestElementCurrent, SrcElementCurrent); | |||
689 | ||||
690 | // Shift the address forward by one element. | |||
691 | llvm::Value *DestElementNext = Builder.CreateConstGEP1_32( | |||
692 | DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); | |||
693 | llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32( | |||
694 | SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); | |||
695 | // Check whether we've reached the end. | |||
696 | llvm::Value *Done = | |||
697 | Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); | |||
698 | Builder.CreateCondBr(Done, DoneBB, BodyBB); | |||
699 | DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock()); | |||
700 | SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock()); | |||
701 | ||||
702 | // Done. | |||
703 | EmitBlock(DoneBB, /*IsFinished=*/true); | |||
704 | } | |||
705 | ||||
706 | void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr, | |||
707 | Address SrcAddr, const VarDecl *DestVD, | |||
708 | const VarDecl *SrcVD, const Expr *Copy) { | |||
709 | if (OriginalType->isArrayType()) { | |||
710 | const auto *BO = dyn_cast<BinaryOperator>(Copy); | |||
711 | if (BO && BO->getOpcode() == BO_Assign) { | |||
712 | // Perform simple memcpy for simple copying. | |||
713 | LValue Dest = MakeAddrLValue(DestAddr, OriginalType); | |||
714 | LValue Src = MakeAddrLValue(SrcAddr, OriginalType); | |||
715 | EmitAggregateAssign(Dest, Src, OriginalType); | |||
716 | } else { | |||
717 | // For arrays with complex element types perform element by element | |||
718 | // copying. | |||
719 | EmitOMPAggregateAssign( | |||
720 | DestAddr, SrcAddr, OriginalType, | |||
721 | [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) { | |||
722 | // Working with the single array element, so have to remap | |||
723 | // destination and source variables to corresponding array | |||
724 | // elements. | |||
725 | CodeGenFunction::OMPPrivateScope Remap(*this); | |||
726 | Remap.addPrivate(DestVD, [DestElement]() { return DestElement; }); | |||
727 | Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; }); | |||
728 | (void)Remap.Privatize(); | |||
729 | EmitIgnoredExpr(Copy); | |||
730 | }); | |||
731 | } | |||
732 | } else { | |||
733 | // Remap pseudo source variable to private copy. | |||
734 | CodeGenFunction::OMPPrivateScope Remap(*this); | |||
735 | Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; }); | |||
736 | Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; }); | |||
737 | (void)Remap.Privatize(); | |||
738 | // Emit copying of the whole variable. | |||
739 | EmitIgnoredExpr(Copy); | |||
740 | } | |||
741 | } | |||
742 | ||||
743 | bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, | |||
744 | OMPPrivateScope &PrivateScope) { | |||
745 | if (!HaveInsertPoint()) | |||
746 | return false; | |||
747 | bool DeviceConstTarget = | |||
748 | getLangOpts().OpenMPIsDevice && | |||
749 | isOpenMPTargetExecutionDirective(D.getDirectiveKind()); | |||
750 | bool FirstprivateIsLastprivate = false; | |||
751 | llvm::DenseSet<const VarDecl *> Lastprivates; | |||
752 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { | |||
753 | for (const auto *D : C->varlists()) | |||
754 | Lastprivates.insert( | |||
755 | cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl()); | |||
756 | } | |||
757 | llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; | |||
758 | llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions; | |||
759 | getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); | |||
760 | // Force emission of the firstprivate copy if the directive does not emit | |||
761 | // outlined function, like omp for, omp simd, omp distribute etc. | |||
762 | bool MustEmitFirstprivateCopy = | |||
763 | CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown; | |||
764 | for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) { | |||
765 | auto IRef = C->varlist_begin(); | |||
766 | auto InitsRef = C->inits().begin(); | |||
767 | for (const Expr *IInit : C->private_copies()) { | |||
768 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
769 | bool ThisFirstprivateIsLastprivate = | |||
770 | Lastprivates.count(OrigVD->getCanonicalDecl()) > 0; | |||
771 | const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD); | |||
772 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); | |||
773 | if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD && | |||
774 | !FD->getType()->isReferenceType() && | |||
775 | (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) { | |||
776 | EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()); | |||
777 | ++IRef; | |||
778 | ++InitsRef; | |||
779 | continue; | |||
780 | } | |||
781 | // Do not emit copy for firstprivate constant variables in target regions, | |||
782 | // captured by reference. | |||
783 | if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) && | |||
784 | FD && FD->getType()->isReferenceType() && | |||
785 | (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) { | |||
786 | (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this, | |||
787 | OrigVD); | |||
788 | ++IRef; | |||
789 | ++InitsRef; | |||
790 | continue; | |||
791 | } | |||
792 | FirstprivateIsLastprivate = | |||
793 | FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate; | |||
794 | if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
795 | const auto *VDInit = | |||
796 | cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); | |||
797 | bool IsRegistered; | |||
798 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), | |||
799 | /*RefersToEnclosingVariableOrCapture=*/FD != nullptr, | |||
800 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); | |||
801 | LValue OriginalLVal; | |||
802 | if (!FD) { | |||
803 | // Check if the firstprivate variable is just a constant value. | |||
804 | ConstantEmission CE = tryEmitAsConstant(&DRE); | |||
805 | if (CE && !CE.isReference()) { | |||
806 | // Constant value, no need to create a copy. | |||
807 | ++IRef; | |||
808 | ++InitsRef; | |||
809 | continue; | |||
810 | } | |||
811 | if (CE && CE.isReference()) { | |||
812 | OriginalLVal = CE.getReferenceLValue(*this, &DRE); | |||
813 | } else { | |||
814 | assert(!CE && "Expected non-constant firstprivate.")((!CE && "Expected non-constant firstprivate.") ? static_cast <void> (0) : __assert_fail ("!CE && \"Expected non-constant firstprivate.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 814, __PRETTY_FUNCTION__)); | |||
815 | OriginalLVal = EmitLValue(&DRE); | |||
816 | } | |||
817 | } else { | |||
818 | OriginalLVal = EmitLValue(&DRE); | |||
819 | } | |||
820 | QualType Type = VD->getType(); | |||
821 | if (Type->isArrayType()) { | |||
822 | // Emit VarDecl with copy init for arrays. | |||
823 | // Get the address of the original variable captured in current | |||
824 | // captured region. | |||
825 | IsRegistered = PrivateScope.addPrivate( | |||
826 | OrigVD, [this, VD, Type, OriginalLVal, VDInit]() { | |||
827 | AutoVarEmission Emission = EmitAutoVarAlloca(*VD); | |||
828 | const Expr *Init = VD->getInit(); | |||
829 | if (!isa<CXXConstructExpr>(Init) || | |||
830 | isTrivialInitializer(Init)) { | |||
831 | // Perform simple memcpy. | |||
832 | LValue Dest = | |||
833 | MakeAddrLValue(Emission.getAllocatedAddress(), Type); | |||
834 | EmitAggregateAssign(Dest, OriginalLVal, Type); | |||
835 | } else { | |||
836 | EmitOMPAggregateAssign( | |||
837 | Emission.getAllocatedAddress(), | |||
838 | OriginalLVal.getAddress(*this), Type, | |||
839 | [this, VDInit, Init](Address DestElement, | |||
840 | Address SrcElement) { | |||
841 | // Clean up any temporaries needed by the | |||
842 | // initialization. | |||
843 | RunCleanupsScope InitScope(*this); | |||
844 | // Emit initialization for single element. | |||
845 | setAddrOfLocalVar(VDInit, SrcElement); | |||
846 | EmitAnyExprToMem(Init, DestElement, | |||
847 | Init->getType().getQualifiers(), | |||
848 | /*IsInitializer*/ false); | |||
849 | LocalDeclMap.erase(VDInit); | |||
850 | }); | |||
851 | } | |||
852 | EmitAutoVarCleanups(Emission); | |||
853 | return Emission.getAllocatedAddress(); | |||
854 | }); | |||
855 | } else { | |||
856 | Address OriginalAddr = OriginalLVal.getAddress(*this); | |||
857 | IsRegistered = PrivateScope.addPrivate( | |||
858 | OrigVD, [this, VDInit, OriginalAddr, VD]() { | |||
859 | // Emit private VarDecl with copy init. | |||
860 | // Remap temp VDInit variable to the address of the original | |||
861 | // variable (for proper handling of captured global variables). | |||
862 | setAddrOfLocalVar(VDInit, OriginalAddr); | |||
863 | EmitDecl(*VD); | |||
864 | LocalDeclMap.erase(VDInit); | |||
865 | return GetAddrOfLocalVar(VD); | |||
866 | }); | |||
867 | } | |||
868 | assert(IsRegistered &&((IsRegistered && "firstprivate var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 869, __PRETTY_FUNCTION__)) | |||
869 | "firstprivate var already registered as private")((IsRegistered && "firstprivate var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 869, __PRETTY_FUNCTION__)); | |||
870 | // Silence the warning about unused variable. | |||
871 | (void)IsRegistered; | |||
872 | } | |||
873 | ++IRef; | |||
874 | ++InitsRef; | |||
875 | } | |||
876 | } | |||
877 | return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty(); | |||
878 | } | |||
879 | ||||
880 | void CodeGenFunction::EmitOMPPrivateClause( | |||
881 | const OMPExecutableDirective &D, | |||
882 | CodeGenFunction::OMPPrivateScope &PrivateScope) { | |||
883 | if (!HaveInsertPoint()) | |||
884 | return; | |||
885 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; | |||
886 | for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) { | |||
887 | auto IRef = C->varlist_begin(); | |||
888 | for (const Expr *IInit : C->private_copies()) { | |||
889 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
890 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
891 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); | |||
892 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { | |||
893 | // Emit private VarDecl with copy init. | |||
894 | EmitDecl(*VD); | |||
895 | return GetAddrOfLocalVar(VD); | |||
896 | }); | |||
897 | assert(IsRegistered && "private var already registered as private")((IsRegistered && "private var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"private var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 897, __PRETTY_FUNCTION__)); | |||
898 | // Silence the warning about unused variable. | |||
899 | (void)IsRegistered; | |||
900 | } | |||
901 | ++IRef; | |||
902 | } | |||
903 | } | |||
904 | } | |||
905 | ||||
906 | bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { | |||
907 | if (!HaveInsertPoint()) | |||
908 | return false; | |||
909 | // threadprivate_var1 = master_threadprivate_var1; | |||
910 | // operator=(threadprivate_var2, master_threadprivate_var2); | |||
911 | // ... | |||
912 | // __kmpc_barrier(&loc, global_tid); | |||
913 | llvm::DenseSet<const VarDecl *> CopiedVars; | |||
914 | llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; | |||
915 | for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) { | |||
916 | auto IRef = C->varlist_begin(); | |||
917 | auto ISrcRef = C->source_exprs().begin(); | |||
918 | auto IDestRef = C->destination_exprs().begin(); | |||
919 | for (const Expr *AssignOp : C->assignment_ops()) { | |||
920 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
921 | QualType Type = VD->getType(); | |||
922 | if (CopiedVars.insert(VD->getCanonicalDecl()).second) { | |||
923 | // Get the address of the master variable. If we are emitting code with | |||
924 | // TLS support, the address is passed from the master as field in the | |||
925 | // captured declaration. | |||
926 | Address MasterAddr = Address::invalid(); | |||
927 | if (getLangOpts().OpenMPUseTLS && | |||
928 | getContext().getTargetInfo().isTLSSupported()) { | |||
929 | assert(CapturedStmtInfo->lookup(VD) &&((CapturedStmtInfo->lookup(VD) && "Copyin threadprivates should have been captured!" ) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo->lookup(VD) && \"Copyin threadprivates should have been captured!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 930, __PRETTY_FUNCTION__)) | |||
930 | "Copyin threadprivates should have been captured!")((CapturedStmtInfo->lookup(VD) && "Copyin threadprivates should have been captured!" ) ? static_cast<void> (0) : __assert_fail ("CapturedStmtInfo->lookup(VD) && \"Copyin threadprivates should have been captured!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 930, __PRETTY_FUNCTION__)); | |||
931 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true, | |||
932 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); | |||
933 | MasterAddr = EmitLValue(&DRE).getAddress(*this); | |||
934 | LocalDeclMap.erase(VD); | |||
935 | } else { | |||
936 | MasterAddr = | |||
937 | Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) | |||
938 | : CGM.GetAddrOfGlobal(VD), | |||
939 | getContext().getDeclAlign(VD)); | |||
940 | } | |||
941 | // Get the address of the threadprivate variable. | |||
942 | Address PrivateAddr = EmitLValue(*IRef).getAddress(*this); | |||
943 | if (CopiedVars.size() == 1) { | |||
944 | // At first check if current thread is a master thread. If it is, no | |||
945 | // need to copy data. | |||
946 | CopyBegin = createBasicBlock("copyin.not.master"); | |||
947 | CopyEnd = createBasicBlock("copyin.not.master.end"); | |||
948 | Builder.CreateCondBr( | |||
949 | Builder.CreateICmpNE( | |||
950 | Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy), | |||
951 | Builder.CreatePtrToInt(PrivateAddr.getPointer(), | |||
952 | CGM.IntPtrTy)), | |||
953 | CopyBegin, CopyEnd); | |||
954 | EmitBlock(CopyBegin); | |||
955 | } | |||
956 | const auto *SrcVD = | |||
957 | cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); | |||
958 | const auto *DestVD = | |||
959 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); | |||
960 | EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp); | |||
961 | } | |||
962 | ++IRef; | |||
963 | ++ISrcRef; | |||
964 | ++IDestRef; | |||
965 | } | |||
966 | } | |||
967 | if (CopyEnd) { | |||
968 | // Exit out of copying procedure for non-master thread. | |||
969 | EmitBlock(CopyEnd, /*IsFinished=*/true); | |||
970 | return true; | |||
971 | } | |||
972 | return false; | |||
973 | } | |||
974 | ||||
975 | bool CodeGenFunction::EmitOMPLastprivateClauseInit( | |||
976 | const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { | |||
977 | if (!HaveInsertPoint()) | |||
978 | return false; | |||
979 | bool HasAtLeastOneLastprivate = false; | |||
980 | llvm::DenseSet<const VarDecl *> SIMDLCVs; | |||
981 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { | |||
982 | const auto *LoopDirective = cast<OMPLoopDirective>(&D); | |||
983 | for (const Expr *C : LoopDirective->counters()) { | |||
984 | SIMDLCVs.insert( | |||
985 | cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl()); | |||
986 | } | |||
987 | } | |||
988 | llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; | |||
989 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { | |||
990 | HasAtLeastOneLastprivate = true; | |||
991 | if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && | |||
992 | !getLangOpts().OpenMPSimd) | |||
993 | break; | |||
994 | auto IRef = C->varlist_begin(); | |||
995 | auto IDestRef = C->destination_exprs().begin(); | |||
996 | for (const Expr *IInit : C->private_copies()) { | |||
997 | // Keep the address of the original variable for future update at the end | |||
998 | // of the loop. | |||
999 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
1000 | // Taskloops do not require additional initialization, it is done in | |||
1001 | // runtime support library. | |||
1002 | if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { | |||
1003 | const auto *DestVD = | |||
1004 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); | |||
1005 | PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() { | |||
1006 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), | |||
1007 | /*RefersToEnclosingVariableOrCapture=*/ | |||
1008 | CapturedStmtInfo->lookup(OrigVD) != nullptr, | |||
1009 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); | |||
1010 | return EmitLValue(&DRE).getAddress(*this); | |||
1011 | }); | |||
1012 | // Check if the variable is also a firstprivate: in this case IInit is | |||
1013 | // not generated. Initialization of this variable will happen in codegen | |||
1014 | // for 'firstprivate' clause. | |||
1015 | if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) { | |||
1016 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); | |||
1017 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { | |||
1018 | // Emit private VarDecl with copy init. | |||
1019 | EmitDecl(*VD); | |||
1020 | return GetAddrOfLocalVar(VD); | |||
1021 | }); | |||
1022 | assert(IsRegistered &&((IsRegistered && "lastprivate var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"lastprivate var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1023, __PRETTY_FUNCTION__)) | |||
1023 | "lastprivate var already registered as private")((IsRegistered && "lastprivate var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"lastprivate var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1023, __PRETTY_FUNCTION__)); | |||
1024 | (void)IsRegistered; | |||
1025 | } | |||
1026 | } | |||
1027 | ++IRef; | |||
1028 | ++IDestRef; | |||
1029 | } | |||
1030 | } | |||
1031 | return HasAtLeastOneLastprivate; | |||
1032 | } | |||
1033 | ||||
1034 | void CodeGenFunction::EmitOMPLastprivateClauseFinal( | |||
1035 | const OMPExecutableDirective &D, bool NoFinals, | |||
1036 | llvm::Value *IsLastIterCond) { | |||
1037 | if (!HaveInsertPoint()) | |||
1038 | return; | |||
1039 | // Emit following code: | |||
1040 | // if (<IsLastIterCond>) { | |||
1041 | // orig_var1 = private_orig_var1; | |||
1042 | // ... | |||
1043 | // orig_varn = private_orig_varn; | |||
1044 | // } | |||
1045 | llvm::BasicBlock *ThenBB = nullptr; | |||
1046 | llvm::BasicBlock *DoneBB = nullptr; | |||
1047 | if (IsLastIterCond) { | |||
1048 | // Emit implicit barrier if at least one lastprivate conditional is found | |||
1049 | // and this is not a simd mode. | |||
1050 | if (!getLangOpts().OpenMPSimd && | |||
1051 | llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(), | |||
1052 | [](const OMPLastprivateClause *C) { | |||
1053 | return C->getKind() == OMPC_LASTPRIVATE_conditional; | |||
1054 | })) { | |||
1055 | CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(), | |||
1056 | OMPD_unknown, | |||
1057 | /*EmitChecks=*/false, | |||
1058 | /*ForceSimpleCall=*/true); | |||
1059 | } | |||
1060 | ThenBB = createBasicBlock(".omp.lastprivate.then"); | |||
1061 | DoneBB = createBasicBlock(".omp.lastprivate.done"); | |||
1062 | Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); | |||
1063 | EmitBlock(ThenBB); | |||
1064 | } | |||
1065 | llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; | |||
1066 | llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates; | |||
1067 | if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) { | |||
1068 | auto IC = LoopDirective->counters().begin(); | |||
1069 | for (const Expr *F : LoopDirective->finals()) { | |||
1070 | const auto *D = | |||
1071 | cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl(); | |||
1072 | if (NoFinals) | |||
1073 | AlreadyEmittedVars.insert(D); | |||
1074 | else | |||
1075 | LoopCountersAndUpdates[D] = F; | |||
1076 | ++IC; | |||
1077 | } | |||
1078 | } | |||
1079 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { | |||
1080 | auto IRef = C->varlist_begin(); | |||
1081 | auto ISrcRef = C->source_exprs().begin(); | |||
1082 | auto IDestRef = C->destination_exprs().begin(); | |||
1083 | for (const Expr *AssignOp : C->assignment_ops()) { | |||
1084 | const auto *PrivateVD = | |||
1085 | cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
1086 | QualType Type = PrivateVD->getType(); | |||
1087 | const auto *CanonicalVD = PrivateVD->getCanonicalDecl(); | |||
1088 | if (AlreadyEmittedVars.insert(CanonicalVD).second) { | |||
1089 | // If lastprivate variable is a loop control variable for loop-based | |||
1090 | // directive, update its value before copyin back to original | |||
1091 | // variable. | |||
1092 | if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) | |||
1093 | EmitIgnoredExpr(FinalExpr); | |||
1094 | const auto *SrcVD = | |||
1095 | cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); | |||
1096 | const auto *DestVD = | |||
1097 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); | |||
1098 | // Get the address of the private variable. | |||
1099 | Address PrivateAddr = GetAddrOfLocalVar(PrivateVD); | |||
1100 | if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>()) | |||
1101 | PrivateAddr = | |||
1102 | Address(Builder.CreateLoad(PrivateAddr), | |||
1103 | getNaturalTypeAlignment(RefTy->getPointeeType())); | |||
1104 | // Store the last value to the private copy in the last iteration. | |||
1105 | if (C->getKind() == OMPC_LASTPRIVATE_conditional) | |||
1106 | CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate( | |||
1107 | *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD, | |||
1108 | (*IRef)->getExprLoc()); | |||
1109 | // Get the address of the original variable. | |||
1110 | Address OriginalAddr = GetAddrOfLocalVar(DestVD); | |||
1111 | EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp); | |||
1112 | } | |||
1113 | ++IRef; | |||
1114 | ++ISrcRef; | |||
1115 | ++IDestRef; | |||
1116 | } | |||
1117 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) | |||
1118 | EmitIgnoredExpr(PostUpdate); | |||
1119 | } | |||
1120 | if (IsLastIterCond) | |||
1121 | EmitBlock(DoneBB, /*IsFinished=*/true); | |||
1122 | } | |||
1123 | ||||
1124 | void CodeGenFunction::EmitOMPReductionClauseInit( | |||
1125 | const OMPExecutableDirective &D, | |||
1126 | CodeGenFunction::OMPPrivateScope &PrivateScope) { | |||
1127 | if (!HaveInsertPoint()) | |||
1128 | return; | |||
1129 | SmallVector<const Expr *, 4> Shareds; | |||
1130 | SmallVector<const Expr *, 4> Privates; | |||
1131 | SmallVector<const Expr *, 4> ReductionOps; | |||
1132 | SmallVector<const Expr *, 4> LHSs; | |||
1133 | SmallVector<const Expr *, 4> RHSs; | |||
1134 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { | |||
1135 | auto IPriv = C->privates().begin(); | |||
1136 | auto IRed = C->reduction_ops().begin(); | |||
1137 | auto ILHS = C->lhs_exprs().begin(); | |||
1138 | auto IRHS = C->rhs_exprs().begin(); | |||
1139 | for (const Expr *Ref : C->varlists()) { | |||
1140 | Shareds.emplace_back(Ref); | |||
1141 | Privates.emplace_back(*IPriv); | |||
1142 | ReductionOps.emplace_back(*IRed); | |||
1143 | LHSs.emplace_back(*ILHS); | |||
1144 | RHSs.emplace_back(*IRHS); | |||
1145 | std::advance(IPriv, 1); | |||
1146 | std::advance(IRed, 1); | |||
1147 | std::advance(ILHS, 1); | |||
1148 | std::advance(IRHS, 1); | |||
1149 | } | |||
1150 | } | |||
1151 | ReductionCodeGen RedCG(Shareds, Privates, ReductionOps); | |||
1152 | unsigned Count = 0; | |||
1153 | auto ILHS = LHSs.begin(); | |||
1154 | auto IRHS = RHSs.begin(); | |||
1155 | auto IPriv = Privates.begin(); | |||
1156 | for (const Expr *IRef : Shareds) { | |||
1157 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl()); | |||
1158 | // Emit private VarDecl with reduction init. | |||
1159 | RedCG.emitSharedLValue(*this, Count); | |||
1160 | RedCG.emitAggregateType(*this, Count); | |||
1161 | AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); | |||
1162 | RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(), | |||
1163 | RedCG.getSharedLValue(Count), | |||
1164 | [&Emission](CodeGenFunction &CGF) { | |||
1165 | CGF.EmitAutoVarInit(Emission); | |||
1166 | return true; | |||
1167 | }); | |||
1168 | EmitAutoVarCleanups(Emission); | |||
1169 | Address BaseAddr = RedCG.adjustPrivateAddress( | |||
1170 | *this, Count, Emission.getAllocatedAddress()); | |||
1171 | bool IsRegistered = PrivateScope.addPrivate( | |||
1172 | RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; }); | |||
1173 | assert(IsRegistered && "private var already registered as private")((IsRegistered && "private var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"private var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1173, __PRETTY_FUNCTION__)); | |||
1174 | // Silence the warning about unused variable. | |||
1175 | (void)IsRegistered; | |||
1176 | ||||
1177 | const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); | |||
1178 | const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); | |||
1179 | QualType Type = PrivateVD->getType(); | |||
1180 | bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef); | |||
1181 | if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { | |||
1182 | // Store the address of the original variable associated with the LHS | |||
1183 | // implicit variable. | |||
1184 | PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() { | |||
1185 | return RedCG.getSharedLValue(Count).getAddress(*this); | |||
1186 | }); | |||
1187 | PrivateScope.addPrivate( | |||
1188 | RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); }); | |||
1189 | } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || | |||
1190 | isa<ArraySubscriptExpr>(IRef)) { | |||
1191 | // Store the address of the original variable associated with the LHS | |||
1192 | // implicit variable. | |||
1193 | PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() { | |||
1194 | return RedCG.getSharedLValue(Count).getAddress(*this); | |||
1195 | }); | |||
1196 | PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() { | |||
1197 | return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD), | |||
1198 | ConvertTypeForMem(RHSVD->getType()), | |||
1199 | "rhs.begin"); | |||
1200 | }); | |||
1201 | } else { | |||
1202 | QualType Type = PrivateVD->getType(); | |||
1203 | bool IsArray = getContext().getAsArrayType(Type) != nullptr; | |||
1204 | Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this); | |||
1205 | // Store the address of the original variable associated with the LHS | |||
1206 | // implicit variable. | |||
1207 | if (IsArray) { | |||
1208 | OriginalAddr = Builder.CreateElementBitCast( | |||
1209 | OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin"); | |||
1210 | } | |||
1211 | PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; }); | |||
1212 | PrivateScope.addPrivate( | |||
1213 | RHSVD, [this, PrivateVD, RHSVD, IsArray]() { | |||
1214 | return IsArray | |||
1215 | ? Builder.CreateElementBitCast( | |||
1216 | GetAddrOfLocalVar(PrivateVD), | |||
1217 | ConvertTypeForMem(RHSVD->getType()), "rhs.begin") | |||
1218 | : GetAddrOfLocalVar(PrivateVD); | |||
1219 | }); | |||
1220 | } | |||
1221 | ++ILHS; | |||
1222 | ++IRHS; | |||
1223 | ++IPriv; | |||
1224 | ++Count; | |||
1225 | } | |||
1226 | } | |||
1227 | ||||
1228 | void CodeGenFunction::EmitOMPReductionClauseFinal( | |||
1229 | const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) { | |||
1230 | if (!HaveInsertPoint()) | |||
1231 | return; | |||
1232 | llvm::SmallVector<const Expr *, 8> Privates; | |||
1233 | llvm::SmallVector<const Expr *, 8> LHSExprs; | |||
1234 | llvm::SmallVector<const Expr *, 8> RHSExprs; | |||
1235 | llvm::SmallVector<const Expr *, 8> ReductionOps; | |||
1236 | bool HasAtLeastOneReduction = false; | |||
1237 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { | |||
1238 | HasAtLeastOneReduction = true; | |||
1239 | Privates.append(C->privates().begin(), C->privates().end()); | |||
1240 | LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); | |||
1241 | RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); | |||
1242 | ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); | |||
1243 | } | |||
1244 | if (HasAtLeastOneReduction) { | |||
1245 | bool WithNowait = D.getSingleClause<OMPNowaitClause>() || | |||
1246 | isOpenMPParallelDirective(D.getDirectiveKind()) || | |||
1247 | ReductionKind == OMPD_simd; | |||
1248 | bool SimpleReduction = ReductionKind == OMPD_simd; | |||
1249 | // Emit nowait reduction if nowait clause is present or directive is a | |||
1250 | // parallel directive (it always has implicit barrier). | |||
1251 | CGM.getOpenMPRuntime().emitReduction( | |||
1252 | *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps, | |||
1253 | {WithNowait, SimpleReduction, ReductionKind}); | |||
1254 | } | |||
1255 | } | |||
1256 | ||||
1257 | static void emitPostUpdateForReductionClause( | |||
1258 | CodeGenFunction &CGF, const OMPExecutableDirective &D, | |||
1259 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { | |||
1260 | if (!CGF.HaveInsertPoint()) | |||
1261 | return; | |||
1262 | llvm::BasicBlock *DoneBB = nullptr; | |||
1263 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { | |||
1264 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) { | |||
1265 | if (!DoneBB) { | |||
1266 | if (llvm::Value *Cond = CondGen(CGF)) { | |||
1267 | // If the first post-update expression is found, emit conditional | |||
1268 | // block if it was requested. | |||
1269 | llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); | |||
1270 | DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); | |||
1271 | CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); | |||
1272 | CGF.EmitBlock(ThenBB); | |||
1273 | } | |||
1274 | } | |||
1275 | CGF.EmitIgnoredExpr(PostUpdate); | |||
1276 | } | |||
1277 | } | |||
1278 | if (DoneBB) | |||
1279 | CGF.EmitBlock(DoneBB, /*IsFinished=*/true); | |||
1280 | } | |||
1281 | ||||
1282 | namespace { | |||
1283 | /// Codegen lambda for appending distribute lower and upper bounds to outlined | |||
1284 | /// parallel function. This is necessary for combined constructs such as | |||
1285 | /// 'distribute parallel for' | |||
1286 | typedef llvm::function_ref<void(CodeGenFunction &, | |||
1287 | const OMPExecutableDirective &, | |||
1288 | llvm::SmallVectorImpl<llvm::Value *> &)> | |||
1289 | CodeGenBoundParametersTy; | |||
1290 | } // anonymous namespace | |||
1291 | ||||
1292 | static void emitCommonOMPParallelDirective( | |||
1293 | CodeGenFunction &CGF, const OMPExecutableDirective &S, | |||
1294 | OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, | |||
1295 | const CodeGenBoundParametersTy &CodeGenBoundParameters) { | |||
1296 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); | |||
1297 | llvm::Function *OutlinedFn = | |||
1298 | CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( | |||
1299 | S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); | |||
1300 | if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) { | |||
1301 | CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); | |||
1302 | llvm::Value *NumThreads = | |||
1303 | CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), | |||
1304 | /*IgnoreResultAssign=*/true); | |||
1305 | CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( | |||
1306 | CGF, NumThreads, NumThreadsClause->getBeginLoc()); | |||
1307 | } | |||
1308 | if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) { | |||
1309 | CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); | |||
1310 | CGF.CGM.getOpenMPRuntime().emitProcBindClause( | |||
1311 | CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); | |||
1312 | } | |||
1313 | const Expr *IfCond = nullptr; | |||
1314 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
1315 | if (C->getNameModifier() == OMPD_unknown || | |||
1316 | C->getNameModifier() == OMPD_parallel) { | |||
1317 | IfCond = C->getCondition(); | |||
1318 | break; | |||
1319 | } | |||
1320 | } | |||
1321 | ||||
1322 | OMPParallelScope Scope(CGF, S); | |||
1323 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; | |||
1324 | // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk | |||
1325 | // lower and upper bounds with the pragma 'for' chunking mechanism. | |||
1326 | // The following lambda takes care of appending the lower and upper bound | |||
1327 | // parameters when necessary | |||
1328 | CodeGenBoundParameters(CGF, S, CapturedVars); | |||
1329 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); | |||
1330 | CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, | |||
1331 | CapturedVars, IfCond); | |||
1332 | } | |||
1333 | ||||
1334 | static void emitEmptyBoundParameters(CodeGenFunction &, | |||
1335 | const OMPExecutableDirective &, | |||
1336 | llvm::SmallVectorImpl<llvm::Value *> &) {} | |||
1337 | ||||
1338 | void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { | |||
1339 | ||||
1340 | if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) { | |||
1341 | // Check if we have any if clause associated with the directive. | |||
1342 | llvm::Value *IfCond = nullptr; | |||
1343 | if (const auto *C = S.getSingleClause<OMPIfClause>()) | |||
1344 | IfCond = EmitScalarExpr(C->getCondition(), | |||
1345 | /*IgnoreResultAssign=*/true); | |||
1346 | ||||
1347 | llvm::Value *NumThreads = nullptr; | |||
1348 | if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) | |||
1349 | NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(), | |||
1350 | /*IgnoreResultAssign=*/true); | |||
1351 | ||||
1352 | ProcBindKind ProcBind = OMP_PROC_BIND_default; | |||
1353 | if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) | |||
1354 | ProcBind = ProcBindClause->getProcBindKind(); | |||
1355 | ||||
1356 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
1357 | ||||
1358 | // The cleanup callback that finalizes all variabels at the given location, | |||
1359 | // thus calls destructors etc. | |||
1360 | auto FiniCB = [this](InsertPointTy IP) { | |||
1361 | CGBuilderTy::InsertPointGuard IPG(Builder); | |||
1362 | assert(IP.getBlock()->end() != IP.getPoint() &&((IP.getBlock()->end() != IP.getPoint() && "OpenMP IR Builder should cause terminated block!" ) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() != IP.getPoint() && \"OpenMP IR Builder should cause terminated block!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1363, __PRETTY_FUNCTION__)) | |||
1363 | "OpenMP IR Builder should cause terminated block!")((IP.getBlock()->end() != IP.getPoint() && "OpenMP IR Builder should cause terminated block!" ) ? static_cast<void> (0) : __assert_fail ("IP.getBlock()->end() != IP.getPoint() && \"OpenMP IR Builder should cause terminated block!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1363, __PRETTY_FUNCTION__)); | |||
1364 | llvm::BasicBlock *IPBB = IP.getBlock(); | |||
1365 | llvm::BasicBlock *DestBB = IPBB->splitBasicBlock(IP.getPoint()); | |||
1366 | IPBB->getTerminator()->eraseFromParent(); | |||
1367 | Builder.SetInsertPoint(IPBB); | |||
1368 | CodeGenFunction::JumpDest Dest = getJumpDestInCurrentScope(DestBB); | |||
1369 | EmitBranchThroughCleanup(Dest); | |||
1370 | }; | |||
1371 | ||||
1372 | // Privatization callback that performs appropriate action for | |||
1373 | // shared/private/firstprivate/lastprivate/copyin/... variables. | |||
1374 | // | |||
1375 | // TODO: This defaults to shared right now. | |||
1376 | auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, | |||
1377 | llvm::Value &Val, llvm::Value *&ReplVal) { | |||
1378 | // The next line is appropriate only for variables (Val) with the | |||
1379 | // data-sharing attribute "shared". | |||
1380 | ReplVal = &Val; | |||
1381 | ||||
1382 | return CodeGenIP; | |||
1383 | }; | |||
1384 | ||||
1385 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); | |||
1386 | const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt(); | |||
1387 | ||||
1388 | auto BodyGenCB = [ParallelRegionBodyStmt, | |||
1389 | this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, | |||
1390 | llvm::BasicBlock &ContinuationBB) { | |||
1391 | auto OldAllocaIP = AllocaInsertPt; | |||
1392 | AllocaInsertPt = &*AllocaIP.getPoint(); | |||
1393 | ||||
1394 | auto OldReturnBlock = ReturnBlock; | |||
1395 | ReturnBlock = getJumpDestInCurrentScope(&ContinuationBB); | |||
1396 | ||||
1397 | llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); | |||
1398 | CodeGenIPBB->splitBasicBlock(CodeGenIP.getPoint()); | |||
1399 | llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator(); | |||
1400 | CodeGenIPBBTI->removeFromParent(); | |||
1401 | ||||
1402 | Builder.SetInsertPoint(CodeGenIPBB); | |||
1403 | ||||
1404 | EmitStmt(ParallelRegionBodyStmt); | |||
1405 | ||||
1406 | Builder.Insert(CodeGenIPBBTI); | |||
1407 | ||||
1408 | AllocaInsertPt = OldAllocaIP; | |||
1409 | ReturnBlock = OldReturnBlock; | |||
1410 | }; | |||
1411 | ||||
1412 | CGCapturedStmtInfo CGSI(*CS, CR_OpenMP); | |||
1413 | CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI); | |||
1414 | Builder.restoreIP(OMPBuilder->CreateParallel(Builder, BodyGenCB, PrivCB, | |||
1415 | FiniCB, IfCond, NumThreads, | |||
1416 | ProcBind, S.hasCancel())); | |||
1417 | return; | |||
1418 | } | |||
1419 | ||||
1420 | // Emit parallel region as a standalone region. | |||
1421 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
1422 | Action.Enter(CGF); | |||
1423 | OMPPrivateScope PrivateScope(CGF); | |||
1424 | bool Copyins = CGF.EmitOMPCopyinClause(S); | |||
1425 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
1426 | if (Copyins) { | |||
1427 | // Emit implicit barrier to synchronize threads and avoid data races on | |||
1428 | // propagation master's thread values of threadprivate variables to local | |||
1429 | // instances of that variables of all other implicit threads. | |||
1430 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( | |||
1431 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, | |||
1432 | /*ForceSimpleCall=*/true); | |||
1433 | } | |||
1434 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
1435 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
1436 | (void)PrivateScope.Privatize(); | |||
1437 | CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); | |||
1438 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); | |||
1439 | }; | |||
1440 | emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen, | |||
1441 | emitEmptyBoundParameters); | |||
1442 | emitPostUpdateForReductionClause(*this, S, | |||
1443 | [](CodeGenFunction &) { return nullptr; }); | |||
1444 | } | |||
1445 | ||||
1446 | static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, | |||
1447 | int MaxLevel, int Level = 0) { | |||
1448 | assert(Level < MaxLevel && "Too deep lookup during loop body codegen.")((Level < MaxLevel && "Too deep lookup during loop body codegen." ) ? static_cast<void> (0) : __assert_fail ("Level < MaxLevel && \"Too deep lookup during loop body codegen.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1448, __PRETTY_FUNCTION__)); | |||
1449 | const Stmt *SimplifiedS = S->IgnoreContainers(); | |||
1450 | if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) { | |||
1451 | PrettyStackTraceLoc CrashInfo( | |||
1452 | CGF.getContext().getSourceManager(), CS->getLBracLoc(), | |||
1453 | "LLVM IR generation of compound statement ('{}')"); | |||
1454 | ||||
1455 | // Keep track of the current cleanup stack depth, including debug scopes. | |||
1456 | CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); | |||
1457 | for (const Stmt *CurStmt : CS->body()) | |||
1458 | emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); | |||
1459 | return; | |||
1460 | } | |||
1461 | if (SimplifiedS == NextLoop) { | |||
1462 | if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) { | |||
1463 | S = For->getBody(); | |||
1464 | } else { | |||
1465 | assert(isa<CXXForRangeStmt>(SimplifiedS) &&((isa<CXXForRangeStmt>(SimplifiedS) && "Expected canonical for loop or range-based for loop." ) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(SimplifiedS) && \"Expected canonical for loop or range-based for loop.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1466, __PRETTY_FUNCTION__)) | |||
1466 | "Expected canonical for loop or range-based for loop.")((isa<CXXForRangeStmt>(SimplifiedS) && "Expected canonical for loop or range-based for loop." ) ? static_cast<void> (0) : __assert_fail ("isa<CXXForRangeStmt>(SimplifiedS) && \"Expected canonical for loop or range-based for loop.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1466, __PRETTY_FUNCTION__)); | |||
1467 | const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS); | |||
1468 | CGF.EmitStmt(CXXFor->getLoopVarStmt()); | |||
1469 | S = CXXFor->getBody(); | |||
1470 | } | |||
1471 | if (Level + 1 < MaxLevel) { | |||
1472 | NextLoop = OMPLoopDirective::tryToFindNextInnerLoop( | |||
1473 | S, /*TryImperfectlyNestedLoops=*/true); | |||
1474 | emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); | |||
1475 | return; | |||
1476 | } | |||
1477 | } | |||
1478 | CGF.EmitStmt(S); | |||
1479 | } | |||
1480 | ||||
1481 | void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D, | |||
1482 | JumpDest LoopExit) { | |||
1483 | RunCleanupsScope BodyScope(*this); | |||
1484 | // Update counters values on current iteration. | |||
1485 | for (const Expr *UE : D.updates()) | |||
1486 | EmitIgnoredExpr(UE); | |||
1487 | // Update the linear variables. | |||
1488 | // In distribute directives only loop counters may be marked as linear, no | |||
1489 | // need to generate the code for them. | |||
1490 | if (!isOpenMPDistributeDirective(D.getDirectiveKind())) { | |||
1491 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { | |||
1492 | for (const Expr *UE : C->updates()) | |||
1493 | EmitIgnoredExpr(UE); | |||
1494 | } | |||
1495 | } | |||
1496 | ||||
1497 | // On a continue in the body, jump to the end. | |||
1498 | JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue"); | |||
1499 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); | |||
1500 | for (const Expr *E : D.finals_conditions()) { | |||
1501 | if (!E) | |||
1502 | continue; | |||
1503 | // Check that loop counter in non-rectangular nest fits into the iteration | |||
1504 | // space. | |||
1505 | llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next"); | |||
1506 | EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(), | |||
1507 | getProfileCount(D.getBody())); | |||
1508 | EmitBlock(NextBB); | |||
1509 | } | |||
1510 | // Emit loop variables for C++ range loops. | |||
1511 | const Stmt *Body = | |||
1512 | D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(); | |||
1513 | // Emit loop body. | |||
1514 | emitBody(*this, Body, | |||
1515 | OMPLoopDirective::tryToFindNextInnerLoop( | |||
1516 | Body, /*TryImperfectlyNestedLoops=*/true), | |||
1517 | D.getCollapsedNumber()); | |||
1518 | ||||
1519 | // The end (updates/cleanups). | |||
1520 | EmitBlock(Continue.getBlock()); | |||
1521 | BreakContinueStack.pop_back(); | |||
1522 | } | |||
1523 | ||||
1524 | void CodeGenFunction::EmitOMPInnerLoop( | |||
1525 | const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, | |||
1526 | const Expr *IncExpr, | |||
1527 | const llvm::function_ref<void(CodeGenFunction &)> BodyGen, | |||
1528 | const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) { | |||
1529 | auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); | |||
1530 | ||||
1531 | // Start the loop with a block that tests the condition. | |||
1532 | auto CondBlock = createBasicBlock("omp.inner.for.cond"); | |||
1533 | EmitBlock(CondBlock); | |||
1534 | const SourceRange R = S.getSourceRange(); | |||
1535 | LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), | |||
1536 | SourceLocToDebugLoc(R.getEnd())); | |||
1537 | ||||
1538 | // If there are any cleanups between here and the loop-exit scope, | |||
1539 | // create a block to stage a loop exit along. | |||
1540 | llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); | |||
1541 | if (RequiresCleanup) | |||
1542 | ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); | |||
1543 | ||||
1544 | llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body"); | |||
1545 | ||||
1546 | // Emit condition. | |||
1547 | EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S)); | |||
1548 | if (ExitBlock != LoopExit.getBlock()) { | |||
1549 | EmitBlock(ExitBlock); | |||
1550 | EmitBranchThroughCleanup(LoopExit); | |||
1551 | } | |||
1552 | ||||
1553 | EmitBlock(LoopBody); | |||
1554 | incrementProfileCounter(&S); | |||
1555 | ||||
1556 | // Create a block for the increment. | |||
1557 | JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); | |||
1558 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); | |||
1559 | ||||
1560 | BodyGen(*this); | |||
1561 | ||||
1562 | // Emit "IV = IV + 1" and a back-edge to the condition block. | |||
1563 | EmitBlock(Continue.getBlock()); | |||
1564 | EmitIgnoredExpr(IncExpr); | |||
1565 | PostIncGen(*this); | |||
1566 | BreakContinueStack.pop_back(); | |||
1567 | EmitBranch(CondBlock); | |||
1568 | LoopStack.pop(); | |||
1569 | // Emit the fall-through block. | |||
1570 | EmitBlock(LoopExit.getBlock()); | |||
1571 | } | |||
1572 | ||||
1573 | bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { | |||
1574 | if (!HaveInsertPoint()) | |||
1575 | return false; | |||
1576 | // Emit inits for the linear variables. | |||
1577 | bool HasLinears = false; | |||
1578 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { | |||
1579 | for (const Expr *Init : C->inits()) { | |||
1580 | HasLinears = true; | |||
1581 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); | |||
1582 | if (const auto *Ref = | |||
1583 | dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) { | |||
1584 | AutoVarEmission Emission = EmitAutoVarAlloca(*VD); | |||
1585 | const auto *OrigVD = cast<VarDecl>(Ref->getDecl()); | |||
1586 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), | |||
1587 | CapturedStmtInfo->lookup(OrigVD) != nullptr, | |||
1588 | VD->getInit()->getType(), VK_LValue, | |||
1589 | VD->getInit()->getExprLoc()); | |||
1590 | EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(), | |||
1591 | VD->getType()), | |||
1592 | /*capturedByInit=*/false); | |||
1593 | EmitAutoVarCleanups(Emission); | |||
1594 | } else { | |||
1595 | EmitVarDecl(*VD); | |||
1596 | } | |||
1597 | } | |||
1598 | // Emit the linear steps for the linear clauses. | |||
1599 | // If a step is not constant, it is pre-calculated before the loop. | |||
1600 | if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep())) | |||
1601 | if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) { | |||
1602 | EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); | |||
1603 | // Emit calculation of the linear step. | |||
1604 | EmitIgnoredExpr(CS); | |||
1605 | } | |||
1606 | } | |||
1607 | return HasLinears; | |||
1608 | } | |||
1609 | ||||
1610 | void CodeGenFunction::EmitOMPLinearClauseFinal( | |||
1611 | const OMPLoopDirective &D, | |||
1612 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { | |||
1613 | if (!HaveInsertPoint()) | |||
1614 | return; | |||
1615 | llvm::BasicBlock *DoneBB = nullptr; | |||
1616 | // Emit the final values of the linear variables. | |||
1617 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { | |||
1618 | auto IC = C->varlist_begin(); | |||
1619 | for (const Expr *F : C->finals()) { | |||
1620 | if (!DoneBB) { | |||
1621 | if (llvm::Value *Cond = CondGen(*this)) { | |||
1622 | // If the first post-update expression is found, emit conditional | |||
1623 | // block if it was requested. | |||
1624 | llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu"); | |||
1625 | DoneBB = createBasicBlock(".omp.linear.pu.done"); | |||
1626 | Builder.CreateCondBr(Cond, ThenBB, DoneBB); | |||
1627 | EmitBlock(ThenBB); | |||
1628 | } | |||
1629 | } | |||
1630 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); | |||
1631 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), | |||
1632 | CapturedStmtInfo->lookup(OrigVD) != nullptr, | |||
1633 | (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); | |||
1634 | Address OrigAddr = EmitLValue(&DRE).getAddress(*this); | |||
1635 | CodeGenFunction::OMPPrivateScope VarScope(*this); | |||
1636 | VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); | |||
1637 | (void)VarScope.Privatize(); | |||
1638 | EmitIgnoredExpr(F); | |||
1639 | ++IC; | |||
1640 | } | |||
1641 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) | |||
1642 | EmitIgnoredExpr(PostUpdate); | |||
1643 | } | |||
1644 | if (DoneBB) | |||
1645 | EmitBlock(DoneBB, /*IsFinished=*/true); | |||
1646 | } | |||
1647 | ||||
1648 | static void emitAlignedClause(CodeGenFunction &CGF, | |||
1649 | const OMPExecutableDirective &D) { | |||
1650 | if (!CGF.HaveInsertPoint()) | |||
1651 | return; | |||
1652 | for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) { | |||
1653 | llvm::APInt ClauseAlignment(64, 0); | |||
1654 | if (const Expr *AlignmentExpr = Clause->getAlignment()) { | |||
1655 | auto *AlignmentCI = | |||
1656 | cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); | |||
1657 | ClauseAlignment = AlignmentCI->getValue(); | |||
1658 | } | |||
1659 | for (const Expr *E : Clause->varlists()) { | |||
1660 | llvm::APInt Alignment(ClauseAlignment); | |||
1661 | if (Alignment == 0) { | |||
1662 | // OpenMP [2.8.1, Description] | |||
1663 | // If no optional parameter is specified, implementation-defined default | |||
1664 | // alignments for SIMD instructions on the target platforms are assumed. | |||
1665 | Alignment = | |||
1666 | CGF.getContext() | |||
1667 | .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( | |||
1668 | E->getType()->getPointeeType())) | |||
1669 | .getQuantity(); | |||
1670 | } | |||
1671 | assert((Alignment == 0 || Alignment.isPowerOf2()) &&(((Alignment == 0 || Alignment.isPowerOf2()) && "alignment is not power of 2" ) ? static_cast<void> (0) : __assert_fail ("(Alignment == 0 || Alignment.isPowerOf2()) && \"alignment is not power of 2\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1672, __PRETTY_FUNCTION__)) | |||
1672 | "alignment is not power of 2")(((Alignment == 0 || Alignment.isPowerOf2()) && "alignment is not power of 2" ) ? static_cast<void> (0) : __assert_fail ("(Alignment == 0 || Alignment.isPowerOf2()) && \"alignment is not power of 2\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1672, __PRETTY_FUNCTION__)); | |||
1673 | if (Alignment != 0) { | |||
1674 | llvm::Value *PtrValue = CGF.EmitScalarExpr(E); | |||
1675 | CGF.EmitAlignmentAssumption( | |||
1676 | PtrValue, E, /*No second loc needed*/ SourceLocation(), | |||
1677 | llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); | |||
1678 | } | |||
1679 | } | |||
1680 | } | |||
1681 | } | |||
1682 | ||||
1683 | void CodeGenFunction::EmitOMPPrivateLoopCounters( | |||
1684 | const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) { | |||
1685 | if (!HaveInsertPoint()) | |||
1686 | return; | |||
1687 | auto I = S.private_counters().begin(); | |||
1688 | for (const Expr *E : S.counters()) { | |||
1689 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
1690 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); | |||
1691 | // Emit var without initialization. | |||
1692 | AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD); | |||
1693 | EmitAutoVarCleanups(VarEmission); | |||
1694 | LocalDeclMap.erase(PrivateVD); | |||
1695 | (void)LoopScope.addPrivate(VD, [&VarEmission]() { | |||
1696 | return VarEmission.getAllocatedAddress(); | |||
1697 | }); | |||
1698 | if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) || | |||
1699 | VD->hasGlobalStorage()) { | |||
1700 | (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() { | |||
1701 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), | |||
1702 | LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD), | |||
1703 | E->getType(), VK_LValue, E->getExprLoc()); | |||
1704 | return EmitLValue(&DRE).getAddress(*this); | |||
1705 | }); | |||
1706 | } else { | |||
1707 | (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() { | |||
1708 | return VarEmission.getAllocatedAddress(); | |||
1709 | }); | |||
1710 | } | |||
1711 | ++I; | |||
1712 | } | |||
1713 | // Privatize extra loop counters used in loops for ordered(n) clauses. | |||
1714 | for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) { | |||
1715 | if (!C->getNumForLoops()) | |||
1716 | continue; | |||
1717 | for (unsigned I = S.getCollapsedNumber(), | |||
1718 | E = C->getLoopNumIterations().size(); | |||
1719 | I < E; ++I) { | |||
1720 | const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I)); | |||
1721 | const auto *VD = cast<VarDecl>(DRE->getDecl()); | |||
1722 | // Override only those variables that can be captured to avoid re-emission | |||
1723 | // of the variables declared within the loops. | |||
1724 | if (DRE->refersToEnclosingVariableOrCapture()) { | |||
1725 | (void)LoopScope.addPrivate(VD, [this, DRE, VD]() { | |||
1726 | return CreateMemTemp(DRE->getType(), VD->getName()); | |||
1727 | }); | |||
1728 | } | |||
1729 | } | |||
1730 | } | |||
1731 | } | |||
1732 | ||||
1733 | static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, | |||
1734 | const Expr *Cond, llvm::BasicBlock *TrueBlock, | |||
1735 | llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { | |||
1736 | if (!CGF.HaveInsertPoint()) | |||
1737 | return; | |||
1738 | { | |||
1739 | CodeGenFunction::OMPPrivateScope PreCondScope(CGF); | |||
1740 | CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); | |||
1741 | (void)PreCondScope.Privatize(); | |||
1742 | // Get initial values of real counters. | |||
1743 | for (const Expr *I : S.inits()) { | |||
1744 | CGF.EmitIgnoredExpr(I); | |||
1745 | } | |||
1746 | } | |||
1747 | // Create temp loop control variables with their init values to support | |||
1748 | // non-rectangular loops. | |||
1749 | CodeGenFunction::OMPMapVars PreCondVars; | |||
1750 | for (const Expr * E: S.dependent_counters()) { | |||
1751 | if (!E) | |||
1752 | continue; | |||
1753 | assert(!E->getType().getNonReferenceType()->isRecordType() &&((!E->getType().getNonReferenceType()->isRecordType() && "dependent counter must not be an iterator.") ? static_cast< void> (0) : __assert_fail ("!E->getType().getNonReferenceType()->isRecordType() && \"dependent counter must not be an iterator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1754, __PRETTY_FUNCTION__)) | |||
1754 | "dependent counter must not be an iterator.")((!E->getType().getNonReferenceType()->isRecordType() && "dependent counter must not be an iterator.") ? static_cast< void> (0) : __assert_fail ("!E->getType().getNonReferenceType()->isRecordType() && \"dependent counter must not be an iterator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1754, __PRETTY_FUNCTION__)); | |||
1755 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
1756 | Address CounterAddr = | |||
1757 | CGF.CreateMemTemp(VD->getType().getNonReferenceType()); | |||
1758 | (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); | |||
1759 | } | |||
1760 | (void)PreCondVars.apply(CGF); | |||
1761 | for (const Expr *E : S.dependent_inits()) { | |||
1762 | if (!E) | |||
1763 | continue; | |||
1764 | CGF.EmitIgnoredExpr(E); | |||
1765 | } | |||
1766 | // Check that loop is executed at least one time. | |||
1767 | CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); | |||
1768 | PreCondVars.restore(CGF); | |||
1769 | } | |||
1770 | ||||
1771 | void CodeGenFunction::EmitOMPLinearClause( | |||
1772 | const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { | |||
1773 | if (!HaveInsertPoint()) | |||
1774 | return; | |||
1775 | llvm::DenseSet<const VarDecl *> SIMDLCVs; | |||
1776 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { | |||
1777 | const auto *LoopDirective = cast<OMPLoopDirective>(&D); | |||
1778 | for (const Expr *C : LoopDirective->counters()) { | |||
1779 | SIMDLCVs.insert( | |||
1780 | cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl()); | |||
1781 | } | |||
1782 | } | |||
1783 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { | |||
1784 | auto CurPrivate = C->privates().begin(); | |||
1785 | for (const Expr *E : C->varlists()) { | |||
1786 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
1787 | const auto *PrivateVD = | |||
1788 | cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl()); | |||
1789 | if (!SIMDLCVs.count(VD->getCanonicalDecl())) { | |||
1790 | bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() { | |||
1791 | // Emit private VarDecl with copy init. | |||
1792 | EmitVarDecl(*PrivateVD); | |||
1793 | return GetAddrOfLocalVar(PrivateVD); | |||
1794 | }); | |||
1795 | assert(IsRegistered && "linear var already registered as private")((IsRegistered && "linear var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"linear var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1795, __PRETTY_FUNCTION__)); | |||
1796 | // Silence the warning about unused variable. | |||
1797 | (void)IsRegistered; | |||
1798 | } else { | |||
1799 | EmitVarDecl(*PrivateVD); | |||
1800 | } | |||
1801 | ++CurPrivate; | |||
1802 | } | |||
1803 | } | |||
1804 | } | |||
1805 | ||||
1806 | static void emitSimdlenSafelenClause(CodeGenFunction &CGF, | |||
1807 | const OMPExecutableDirective &D, | |||
1808 | bool IsMonotonic) { | |||
1809 | if (!CGF.HaveInsertPoint()) | |||
1810 | return; | |||
1811 | if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) { | |||
1812 | RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), | |||
1813 | /*ignoreResult=*/true); | |||
1814 | auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); | |||
1815 | CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); | |||
1816 | // In presence of finite 'safelen', it may be unsafe to mark all | |||
1817 | // the memory instructions parallel, because loop-carried | |||
1818 | // dependences of 'safelen' iterations are possible. | |||
1819 | if (!IsMonotonic) | |||
1820 | CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); | |||
1821 | } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) { | |||
1822 | RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), | |||
1823 | /*ignoreResult=*/true); | |||
1824 | auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); | |||
1825 | CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); | |||
1826 | // In presence of finite 'safelen', it may be unsafe to mark all | |||
1827 | // the memory instructions parallel, because loop-carried | |||
1828 | // dependences of 'safelen' iterations are possible. | |||
1829 | CGF.LoopStack.setParallel(/*Enable=*/false); | |||
1830 | } | |||
1831 | } | |||
1832 | ||||
1833 | void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D, | |||
1834 | bool IsMonotonic) { | |||
1835 | // Walk clauses and process safelen/lastprivate. | |||
1836 | LoopStack.setParallel(!IsMonotonic); | |||
1837 | LoopStack.setVectorizeEnable(); | |||
1838 | emitSimdlenSafelenClause(*this, D, IsMonotonic); | |||
1839 | } | |||
1840 | ||||
1841 | void CodeGenFunction::EmitOMPSimdFinal( | |||
1842 | const OMPLoopDirective &D, | |||
1843 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { | |||
1844 | if (!HaveInsertPoint()) | |||
1845 | return; | |||
1846 | llvm::BasicBlock *DoneBB = nullptr; | |||
1847 | auto IC = D.counters().begin(); | |||
1848 | auto IPC = D.private_counters().begin(); | |||
1849 | for (const Expr *F : D.finals()) { | |||
1850 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); | |||
1851 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl()); | |||
1852 | const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD); | |||
1853 | if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) || | |||
1854 | OrigVD->hasGlobalStorage() || CED) { | |||
1855 | if (!DoneBB) { | |||
1856 | if (llvm::Value *Cond = CondGen(*this)) { | |||
1857 | // If the first post-update expression is found, emit conditional | |||
1858 | // block if it was requested. | |||
1859 | llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then"); | |||
1860 | DoneBB = createBasicBlock(".omp.final.done"); | |||
1861 | Builder.CreateCondBr(Cond, ThenBB, DoneBB); | |||
1862 | EmitBlock(ThenBB); | |||
1863 | } | |||
1864 | } | |||
1865 | Address OrigAddr = Address::invalid(); | |||
1866 | if (CED) { | |||
1867 | OrigAddr = | |||
1868 | EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this); | |||
1869 | } else { | |||
1870 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD), | |||
1871 | /*RefersToEnclosingVariableOrCapture=*/false, | |||
1872 | (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc()); | |||
1873 | OrigAddr = EmitLValue(&DRE).getAddress(*this); | |||
1874 | } | |||
1875 | OMPPrivateScope VarScope(*this); | |||
1876 | VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); | |||
1877 | (void)VarScope.Privatize(); | |||
1878 | EmitIgnoredExpr(F); | |||
1879 | } | |||
1880 | ++IC; | |||
1881 | ++IPC; | |||
1882 | } | |||
1883 | if (DoneBB) | |||
1884 | EmitBlock(DoneBB, /*IsFinished=*/true); | |||
1885 | } | |||
1886 | ||||
1887 | static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, | |||
1888 | const OMPLoopDirective &S, | |||
1889 | CodeGenFunction::JumpDest LoopExit) { | |||
1890 | CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(CGF, S); | |||
1891 | CGF.EmitOMPLoopBody(S, LoopExit); | |||
1892 | CGF.EmitStopPoint(&S); | |||
1893 | } | |||
1894 | ||||
1895 | /// Emit a helper variable and return corresponding lvalue. | |||
1896 | static LValue EmitOMPHelperVar(CodeGenFunction &CGF, | |||
1897 | const DeclRefExpr *Helper) { | |||
1898 | auto VDecl = cast<VarDecl>(Helper->getDecl()); | |||
1899 | CGF.EmitVarDecl(*VDecl); | |||
1900 | return CGF.EmitLValue(Helper); | |||
1901 | } | |||
1902 | ||||
1903 | static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, | |||
1904 | const RegionCodeGenTy &SimdInitGen, | |||
1905 | const RegionCodeGenTy &BodyCodeGen) { | |||
1906 | auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, | |||
1907 | PrePostActionTy &) { | |||
1908 | CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); | |||
1909 | CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); | |||
1910 | SimdInitGen(CGF); | |||
1911 | ||||
1912 | BodyCodeGen(CGF); | |||
1913 | }; | |||
1914 | auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { | |||
1915 | CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); | |||
1916 | CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); | |||
1917 | ||||
1918 | BodyCodeGen(CGF); | |||
1919 | }; | |||
1920 | const Expr *IfCond = nullptr; | |||
1921 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
1922 | if (CGF.getLangOpts().OpenMP >= 50 && | |||
1923 | (C->getNameModifier() == OMPD_unknown || | |||
1924 | C->getNameModifier() == OMPD_simd)) { | |||
1925 | IfCond = C->getCondition(); | |||
1926 | break; | |||
1927 | } | |||
1928 | } | |||
1929 | if (IfCond) { | |||
1930 | CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); | |||
1931 | } else { | |||
1932 | RegionCodeGenTy ThenRCG(ThenGen); | |||
1933 | ThenRCG(CGF); | |||
1934 | } | |||
1935 | } | |||
1936 | ||||
1937 | static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, | |||
1938 | PrePostActionTy &Action) { | |||
1939 | Action.Enter(CGF); | |||
1940 | assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&((isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive" ) ? static_cast<void> (0) : __assert_fail ("isOpenMPSimdDirective(S.getDirectiveKind()) && \"Expected simd directive\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1941, __PRETTY_FUNCTION__)) | |||
1941 | "Expected simd directive")((isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive" ) ? static_cast<void> (0) : __assert_fail ("isOpenMPSimdDirective(S.getDirectiveKind()) && \"Expected simd directive\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 1941, __PRETTY_FUNCTION__)); | |||
1942 | OMPLoopScope PreInitScope(CGF, S); | |||
1943 | // if (PreCond) { | |||
1944 | // for (IV in 0..LastIteration) BODY; | |||
1945 | // <Final counter/linear vars updates>; | |||
1946 | // } | |||
1947 | // | |||
1948 | if (isOpenMPDistributeDirective(S.getDirectiveKind()) || | |||
1949 | isOpenMPWorksharingDirective(S.getDirectiveKind()) || | |||
1950 | isOpenMPTaskLoopDirective(S.getDirectiveKind())) { | |||
1951 | (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); | |||
1952 | (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); | |||
1953 | } | |||
1954 | ||||
1955 | // Emit: if (PreCond) - begin. | |||
1956 | // If the condition constant folds and can be elided, avoid emitting the | |||
1957 | // whole loop. | |||
1958 | bool CondConstant; | |||
1959 | llvm::BasicBlock *ContBlock = nullptr; | |||
1960 | if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { | |||
1961 | if (!CondConstant) | |||
1962 | return; | |||
1963 | } else { | |||
1964 | llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); | |||
1965 | ContBlock = CGF.createBasicBlock("simd.if.end"); | |||
1966 | emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, | |||
1967 | CGF.getProfileCount(&S)); | |||
1968 | CGF.EmitBlock(ThenBlock); | |||
1969 | CGF.incrementProfileCounter(&S); | |||
1970 | } | |||
1971 | ||||
1972 | // Emit the loop iteration variable. | |||
1973 | const Expr *IVExpr = S.getIterationVariable(); | |||
1974 | const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); | |||
1975 | CGF.EmitVarDecl(*IVDecl); | |||
1976 | CGF.EmitIgnoredExpr(S.getInit()); | |||
1977 | ||||
1978 | // Emit the iterations count variable. | |||
1979 | // If it is not a variable, Sema decided to calculate iterations count on | |||
1980 | // each iteration (e.g., it is foldable into a constant). | |||
1981 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { | |||
1982 | CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); | |||
1983 | // Emit calculation of the iterations count. | |||
1984 | CGF.EmitIgnoredExpr(S.getCalcLastIteration()); | |||
1985 | } | |||
1986 | ||||
1987 | emitAlignedClause(CGF, S); | |||
1988 | (void)CGF.EmitOMPLinearClauseInit(S); | |||
1989 | { | |||
1990 | CodeGenFunction::OMPPrivateScope LoopScope(CGF); | |||
1991 | CGF.EmitOMPPrivateLoopCounters(S, LoopScope); | |||
1992 | CGF.EmitOMPLinearClause(S, LoopScope); | |||
1993 | CGF.EmitOMPPrivateClause(S, LoopScope); | |||
1994 | CGF.EmitOMPReductionClauseInit(S, LoopScope); | |||
1995 | CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion( | |||
1996 | CGF, S, CGF.EmitLValue(S.getIterationVariable())); | |||
1997 | bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); | |||
1998 | (void)LoopScope.Privatize(); | |||
1999 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
2000 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); | |||
2001 | ||||
2002 | emitCommonSimdLoop( | |||
2003 | CGF, S, | |||
2004 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2005 | CGF.EmitOMPSimdInit(S); | |||
2006 | }, | |||
2007 | [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2008 | CGF.EmitOMPInnerLoop( | |||
2009 | S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), | |||
2010 | [&S](CodeGenFunction &CGF) { | |||
2011 | CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter( | |||
2012 | CGF, S); | |||
2013 | CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); | |||
2014 | CGF.EmitStopPoint(&S); | |||
2015 | }, | |||
2016 | [](CodeGenFunction &) {}); | |||
2017 | }); | |||
2018 | CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); | |||
2019 | // Emit final copy of the lastprivate variables at the end of loops. | |||
2020 | if (HasLastprivateClause) | |||
2021 | CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); | |||
2022 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); | |||
2023 | emitPostUpdateForReductionClause(CGF, S, | |||
2024 | [](CodeGenFunction &) { return nullptr; }); | |||
2025 | } | |||
2026 | CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); | |||
2027 | // Emit: if (PreCond) - end. | |||
2028 | if (ContBlock) { | |||
2029 | CGF.EmitBranch(ContBlock); | |||
2030 | CGF.EmitBlock(ContBlock, true); | |||
2031 | } | |||
2032 | } | |||
2033 | ||||
2034 | void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { | |||
2035 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
2036 | emitOMPSimdRegion(CGF, S, Action); | |||
2037 | }; | |||
2038 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2039 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); | |||
2040 | } | |||
2041 | ||||
2042 | void CodeGenFunction::EmitOMPOuterLoop( | |||
2043 | bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S, | |||
2044 | CodeGenFunction::OMPPrivateScope &LoopScope, | |||
2045 | const CodeGenFunction::OMPLoopArguments &LoopArgs, | |||
2046 | const CodeGenFunction::CodeGenLoopTy &CodeGenLoop, | |||
2047 | const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) { | |||
2048 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); | |||
2049 | ||||
2050 | const Expr *IVExpr = S.getIterationVariable(); | |||
2051 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); | |||
2052 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); | |||
2053 | ||||
2054 | JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); | |||
2055 | ||||
2056 | // Start the loop with a block that tests the condition. | |||
2057 | llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond"); | |||
2058 | EmitBlock(CondBlock); | |||
2059 | const SourceRange R = S.getSourceRange(); | |||
2060 | LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), | |||
2061 | SourceLocToDebugLoc(R.getEnd())); | |||
2062 | ||||
2063 | llvm::Value *BoolCondVal = nullptr; | |||
2064 | if (!DynamicOrOrdered) { | |||
2065 | // UB = min(UB, GlobalUB) or | |||
2066 | // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g. | |||
2067 | // 'distribute parallel for') | |||
2068 | EmitIgnoredExpr(LoopArgs.EUB); | |||
2069 | // IV = LB | |||
2070 | EmitIgnoredExpr(LoopArgs.Init); | |||
2071 | // IV < UB | |||
2072 | BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond); | |||
2073 | } else { | |||
2074 | BoolCondVal = | |||
2075 | RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL, | |||
2076 | LoopArgs.LB, LoopArgs.UB, LoopArgs.ST); | |||
2077 | } | |||
2078 | ||||
2079 | // If there are any cleanups between here and the loop-exit scope, | |||
2080 | // create a block to stage a loop exit along. | |||
2081 | llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); | |||
2082 | if (LoopScope.requiresCleanups()) | |||
2083 | ExitBlock = createBasicBlock("omp.dispatch.cleanup"); | |||
2084 | ||||
2085 | llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body"); | |||
2086 | Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); | |||
2087 | if (ExitBlock != LoopExit.getBlock()) { | |||
2088 | EmitBlock(ExitBlock); | |||
2089 | EmitBranchThroughCleanup(LoopExit); | |||
2090 | } | |||
2091 | EmitBlock(LoopBody); | |||
2092 | ||||
2093 | // Emit "IV = LB" (in case of static schedule, we have already calculated new | |||
2094 | // LB for loop condition and emitted it above). | |||
2095 | if (DynamicOrOrdered) | |||
2096 | EmitIgnoredExpr(LoopArgs.Init); | |||
2097 | ||||
2098 | // Create a block for the increment. | |||
2099 | JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); | |||
2100 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); | |||
2101 | ||||
2102 | emitCommonSimdLoop( | |||
2103 | *this, S, | |||
2104 | [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2105 | // Generate !llvm.loop.parallel metadata for loads and stores for loops | |||
2106 | // with dynamic/guided scheduling and without ordered clause. | |||
2107 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) | |||
2108 | CGF.LoopStack.setParallel(!IsMonotonic); | |||
2109 | else | |||
2110 | CGF.EmitOMPSimdInit(S, IsMonotonic); | |||
2111 | }, | |||
2112 | [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered, | |||
2113 | &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2114 | SourceLocation Loc = S.getBeginLoc(); | |||
2115 | // when 'distribute' is not combined with a 'for': | |||
2116 | // while (idx <= UB) { BODY; ++idx; } | |||
2117 | // when 'distribute' is combined with a 'for' | |||
2118 | // (e.g. 'distribute parallel for') | |||
2119 | // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; } | |||
2120 | CGF.EmitOMPInnerLoop( | |||
2121 | S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr, | |||
2122 | [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { | |||
2123 | CodeGenLoop(CGF, S, LoopExit); | |||
2124 | }, | |||
2125 | [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { | |||
2126 | CodeGenOrdered(CGF, Loc, IVSize, IVSigned); | |||
2127 | }); | |||
2128 | }); | |||
2129 | ||||
2130 | EmitBlock(Continue.getBlock()); | |||
2131 | BreakContinueStack.pop_back(); | |||
2132 | if (!DynamicOrOrdered) { | |||
2133 | // Emit "LB = LB + Stride", "UB = UB + Stride". | |||
2134 | EmitIgnoredExpr(LoopArgs.NextLB); | |||
2135 | EmitIgnoredExpr(LoopArgs.NextUB); | |||
2136 | } | |||
2137 | ||||
2138 | EmitBranch(CondBlock); | |||
2139 | LoopStack.pop(); | |||
2140 | // Emit the fall-through block. | |||
2141 | EmitBlock(LoopExit.getBlock()); | |||
2142 | ||||
2143 | // Tell the runtime we are done. | |||
2144 | auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { | |||
2145 | if (!DynamicOrOrdered) | |||
2146 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), | |||
2147 | S.getDirectiveKind()); | |||
2148 | }; | |||
2149 | OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); | |||
2150 | } | |||
2151 | ||||
2152 | void CodeGenFunction::EmitOMPForOuterLoop( | |||
2153 | const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic, | |||
2154 | const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, | |||
2155 | const OMPLoopArguments &LoopArgs, | |||
2156 | const CodeGenDispatchBoundsTy &CGDispatchBounds) { | |||
2157 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); | |||
2158 | ||||
2159 | // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). | |||
2160 | const bool DynamicOrOrdered = | |||
2161 | Ordered || RT.isDynamic(ScheduleKind.Schedule); | |||
2162 | ||||
2163 | assert((Ordered ||(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs .Chunk != nullptr)) && "static non-chunked schedule does not need outer loop" ) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 2166, __PRETTY_FUNCTION__)) | |||
2164 | !RT.isStaticNonchunked(ScheduleKind.Schedule,(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs .Chunk != nullptr)) && "static non-chunked schedule does not need outer loop" ) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 2166, __PRETTY_FUNCTION__)) | |||
2165 | LoopArgs.Chunk != nullptr)) &&(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs .Chunk != nullptr)) && "static non-chunked schedule does not need outer loop" ) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 2166, __PRETTY_FUNCTION__)) | |||
2166 | "static non-chunked schedule does not need outer loop")(((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs .Chunk != nullptr)) && "static non-chunked schedule does not need outer loop" ) ? static_cast<void> (0) : __assert_fail ("(Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, LoopArgs.Chunk != nullptr)) && \"static non-chunked schedule does not need outer loop\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 2166, __PRETTY_FUNCTION__)); | |||
2167 | ||||
2168 | // Emit outer loop. | |||
2169 | // | |||
2170 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] | |||
2171 | // When schedule(dynamic,chunk_size) is specified, the iterations are | |||
2172 | // distributed to threads in the team in chunks as the threads request them. | |||
2173 | // Each thread executes a chunk of iterations, then requests another chunk, | |||
2174 | // until no chunks remain to be distributed. Each chunk contains chunk_size | |||
2175 | // iterations, except for the last chunk to be distributed, which may have | |||
2176 | // fewer iterations. When no chunk_size is specified, it defaults to 1. | |||
2177 | // | |||
2178 | // When schedule(guided,chunk_size) is specified, the iterations are assigned | |||
2179 | // to threads in the team in chunks as the executing threads request them. | |||
2180 | // Each thread executes a chunk of iterations, then requests another chunk, | |||
2181 | // until no chunks remain to be assigned. For a chunk_size of 1, the size of | |||
2182 | // each chunk is proportional to the number of unassigned iterations divided | |||
2183 | // by the number of threads in the team, decreasing to 1. For a chunk_size | |||
2184 | // with value k (greater than 1), the size of each chunk is determined in the | |||
2185 | // same way, with the restriction that the chunks do not contain fewer than k | |||
2186 | // iterations (except for the last chunk to be assigned, which may have fewer | |||
2187 | // than k iterations). | |||
2188 | // | |||
2189 | // When schedule(auto) is specified, the decision regarding scheduling is | |||
2190 | // delegated to the compiler and/or runtime system. The programmer gives the | |||
2191 | // implementation the freedom to choose any possible mapping of iterations to | |||
2192 | // threads in the team. | |||
2193 | // | |||
2194 | // When schedule(runtime) is specified, the decision regarding scheduling is | |||
2195 | // deferred until run time, and the schedule and chunk size are taken from the | |||
2196 | // run-sched-var ICV. If the ICV is set to auto, the schedule is | |||
2197 | // implementation defined | |||
2198 | // | |||
2199 | // while(__kmpc_dispatch_next(&LB, &UB)) { | |||
2200 | // idx = LB; | |||
2201 | // while (idx <= UB) { BODY; ++idx; | |||
2202 | // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. | |||
2203 | // } // inner loop | |||
2204 | // } | |||
2205 | // | |||
2206 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] | |||
2207 | // When schedule(static, chunk_size) is specified, iterations are divided into | |||
2208 | // chunks of size chunk_size, and the chunks are assigned to the threads in | |||
2209 | // the team in a round-robin fashion in the order of the thread number. | |||
2210 | // | |||
2211 | // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { | |||
2212 | // while (idx <= UB) { BODY; ++idx; } // inner loop | |||
2213 | // LB = LB + ST; | |||
2214 | // UB = UB + ST; | |||
2215 | // } | |||
2216 | // | |||
2217 | ||||
2218 | const Expr *IVExpr = S.getIterationVariable(); | |||
2219 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); | |||
2220 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); | |||
2221 | ||||
2222 | if (DynamicOrOrdered) { | |||
2223 | const std::pair<llvm::Value *, llvm::Value *> DispatchBounds = | |||
2224 | CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); | |||
2225 | llvm::Value *LBVal = DispatchBounds.first; | |||
2226 | llvm::Value *UBVal = DispatchBounds.second; | |||
2227 | CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal, | |||
2228 | LoopArgs.Chunk}; | |||
2229 | RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize, | |||
2230 | IVSigned, Ordered, DipatchRTInputValues); | |||
2231 | } else { | |||
2232 | CGOpenMPRuntime::StaticRTInput StaticInit( | |||
2233 | IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, | |||
2234 | LoopArgs.ST, LoopArgs.Chunk); | |||
2235 | RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(), | |||
2236 | ScheduleKind, StaticInit); | |||
2237 | } | |||
2238 | ||||
2239 | auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, | |||
2240 | const unsigned IVSize, | |||
2241 | const bool IVSigned) { | |||
2242 | if (Ordered) { | |||
2243 | CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, | |||
2244 | IVSigned); | |||
2245 | } | |||
2246 | }; | |||
2247 | ||||
2248 | OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, | |||
2249 | LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB); | |||
2250 | OuterLoopArgs.IncExpr = S.getInc(); | |||
2251 | OuterLoopArgs.Init = S.getInit(); | |||
2252 | OuterLoopArgs.Cond = S.getCond(); | |||
2253 | OuterLoopArgs.NextLB = S.getNextLowerBound(); | |||
2254 | OuterLoopArgs.NextUB = S.getNextUpperBound(); | |||
2255 | EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs, | |||
2256 | emitOMPLoopBodyWithStopPoint, CodeGenOrdered); | |||
2257 | } | |||
2258 | ||||
2259 | static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, | |||
2260 | const unsigned IVSize, const bool IVSigned) {} | |||
2261 | ||||
2262 | void CodeGenFunction::EmitOMPDistributeOuterLoop( | |||
2263 | OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S, | |||
2264 | OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs, | |||
2265 | const CodeGenLoopTy &CodeGenLoopContent) { | |||
2266 | ||||
2267 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); | |||
2268 | ||||
2269 | // Emit outer loop. | |||
2270 | // Same behavior as a OMPForOuterLoop, except that schedule cannot be | |||
2271 | // dynamic | |||
2272 | // | |||
2273 | ||||
2274 | const Expr *IVExpr = S.getIterationVariable(); | |||
2275 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); | |||
2276 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); | |||
2277 | ||||
2278 | CGOpenMPRuntime::StaticRTInput StaticInit( | |||
2279 | IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, | |||
2280 | LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); | |||
2281 | RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit); | |||
2282 | ||||
2283 | // for combined 'distribute' and 'for' the increment expression of distribute | |||
2284 | // is stored in DistInc. For 'distribute' alone, it is in Inc. | |||
2285 | Expr *IncExpr; | |||
2286 | if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())) | |||
2287 | IncExpr = S.getDistInc(); | |||
2288 | else | |||
2289 | IncExpr = S.getInc(); | |||
2290 | ||||
2291 | // this routine is shared by 'omp distribute parallel for' and | |||
2292 | // 'omp distribute': select the right EUB expression depending on the | |||
2293 | // directive | |||
2294 | OMPLoopArguments OuterLoopArgs; | |||
2295 | OuterLoopArgs.LB = LoopArgs.LB; | |||
2296 | OuterLoopArgs.UB = LoopArgs.UB; | |||
2297 | OuterLoopArgs.ST = LoopArgs.ST; | |||
2298 | OuterLoopArgs.IL = LoopArgs.IL; | |||
2299 | OuterLoopArgs.Chunk = LoopArgs.Chunk; | |||
2300 | OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
2301 | ? S.getCombinedEnsureUpperBound() | |||
2302 | : S.getEnsureUpperBound(); | |||
2303 | OuterLoopArgs.IncExpr = IncExpr; | |||
2304 | OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
2305 | ? S.getCombinedInit() | |||
2306 | : S.getInit(); | |||
2307 | OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
2308 | ? S.getCombinedCond() | |||
2309 | : S.getCond(); | |||
2310 | OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
2311 | ? S.getCombinedNextLowerBound() | |||
2312 | : S.getNextLowerBound(); | |||
2313 | OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
2314 | ? S.getCombinedNextUpperBound() | |||
2315 | : S.getNextUpperBound(); | |||
2316 | ||||
2317 | EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, | |||
2318 | LoopScope, OuterLoopArgs, CodeGenLoopContent, | |||
2319 | emitEmptyOrdered); | |||
2320 | } | |||
2321 | ||||
2322 | static std::pair<LValue, LValue> | |||
2323 | emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, | |||
2324 | const OMPExecutableDirective &S) { | |||
2325 | const OMPLoopDirective &LS = cast<OMPLoopDirective>(S); | |||
2326 | LValue LB = | |||
2327 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); | |||
2328 | LValue UB = | |||
2329 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); | |||
2330 | ||||
2331 | // When composing 'distribute' with 'for' (e.g. as in 'distribute | |||
2332 | // parallel for') we need to use the 'distribute' | |||
2333 | // chunk lower and upper bounds rather than the whole loop iteration | |||
2334 | // space. These are parameters to the outlined function for 'parallel' | |||
2335 | // and we copy the bounds of the previous schedule into the | |||
2336 | // the current ones. | |||
2337 | LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); | |||
2338 | LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); | |||
2339 | llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( | |||
2340 | PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc()); | |||
2341 | PrevLBVal = CGF.EmitScalarConversion( | |||
2342 | PrevLBVal, LS.getPrevLowerBoundVariable()->getType(), | |||
2343 | LS.getIterationVariable()->getType(), | |||
2344 | LS.getPrevLowerBoundVariable()->getExprLoc()); | |||
2345 | llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( | |||
2346 | PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc()); | |||
2347 | PrevUBVal = CGF.EmitScalarConversion( | |||
2348 | PrevUBVal, LS.getPrevUpperBoundVariable()->getType(), | |||
2349 | LS.getIterationVariable()->getType(), | |||
2350 | LS.getPrevUpperBoundVariable()->getExprLoc()); | |||
2351 | ||||
2352 | CGF.EmitStoreOfScalar(PrevLBVal, LB); | |||
2353 | CGF.EmitStoreOfScalar(PrevUBVal, UB); | |||
2354 | ||||
2355 | return {LB, UB}; | |||
2356 | } | |||
2357 | ||||
2358 | /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then | |||
2359 | /// we need to use the LB and UB expressions generated by the worksharing | |||
2360 | /// code generation support, whereas in non combined situations we would | |||
2361 | /// just emit 0 and the LastIteration expression | |||
2362 | /// This function is necessary due to the difference of the LB and UB | |||
2363 | /// types for the RT emission routines for 'for_static_init' and | |||
2364 | /// 'for_dispatch_init' | |||
2365 | static std::pair<llvm::Value *, llvm::Value *> | |||
2366 | emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, | |||
2367 | const OMPExecutableDirective &S, | |||
2368 | Address LB, Address UB) { | |||
2369 | const OMPLoopDirective &LS = cast<OMPLoopDirective>(S); | |||
2370 | const Expr *IVExpr = LS.getIterationVariable(); | |||
2371 | // when implementing a dynamic schedule for a 'for' combined with a | |||
2372 | // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop | |||
2373 | // is not normalized as each team only executes its own assigned | |||
2374 | // distribute chunk | |||
2375 | QualType IteratorTy = IVExpr->getType(); | |||
2376 | llvm::Value *LBVal = | |||
2377 | CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); | |||
2378 | llvm::Value *UBVal = | |||
2379 | CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); | |||
2380 | return {LBVal, UBVal}; | |||
2381 | } | |||
2382 | ||||
2383 | static void emitDistributeParallelForDistributeInnerBoundParams( | |||
2384 | CodeGenFunction &CGF, const OMPExecutableDirective &S, | |||
2385 | llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) { | |||
2386 | const auto &Dir = cast<OMPLoopDirective>(S); | |||
2387 | LValue LB = | |||
2388 | CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); | |||
2389 | llvm::Value *LBCast = | |||
2390 | CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), | |||
2391 | CGF.SizeTy, /*isSigned=*/false); | |||
2392 | CapturedVars.push_back(LBCast); | |||
2393 | LValue UB = | |||
2394 | CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); | |||
2395 | ||||
2396 | llvm::Value *UBCast = | |||
2397 | CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), | |||
2398 | CGF.SizeTy, /*isSigned=*/false); | |||
2399 | CapturedVars.push_back(UBCast); | |||
2400 | } | |||
2401 | ||||
2402 | static void | |||
2403 | emitInnerParallelForWhenCombined(CodeGenFunction &CGF, | |||
2404 | const OMPLoopDirective &S, | |||
2405 | CodeGenFunction::JumpDest LoopExit) { | |||
2406 | auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, | |||
2407 | PrePostActionTy &Action) { | |||
2408 | Action.Enter(CGF); | |||
2409 | bool HasCancel = false; | |||
2410 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) { | |||
2411 | if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S)) | |||
2412 | HasCancel = D->hasCancel(); | |||
2413 | else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S)) | |||
2414 | HasCancel = D->hasCancel(); | |||
2415 | else if (const auto *D = | |||
2416 | dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S)) | |||
2417 | HasCancel = D->hasCancel(); | |||
2418 | } | |||
2419 | CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), | |||
2420 | HasCancel); | |||
2421 | CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), | |||
2422 | emitDistributeParallelForInnerBounds, | |||
2423 | emitDistributeParallelForDispatchBounds); | |||
2424 | }; | |||
2425 | ||||
2426 | emitCommonOMPParallelDirective( | |||
2427 | CGF, S, | |||
2428 | isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for, | |||
2429 | CGInlinedWorksharingLoop, | |||
2430 | emitDistributeParallelForDistributeInnerBoundParams); | |||
2431 | } | |||
2432 | ||||
2433 | void CodeGenFunction::EmitOMPDistributeParallelForDirective( | |||
2434 | const OMPDistributeParallelForDirective &S) { | |||
2435 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2436 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
2437 | S.getDistInc()); | |||
2438 | }; | |||
2439 | OMPLexicalScope Scope(*this, S, OMPD_parallel); | |||
2440 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); | |||
2441 | } | |||
2442 | ||||
2443 | void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective( | |||
2444 | const OMPDistributeParallelForSimdDirective &S) { | |||
2445 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2446 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
2447 | S.getDistInc()); | |||
2448 | }; | |||
2449 | OMPLexicalScope Scope(*this, S, OMPD_parallel); | |||
2450 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); | |||
2451 | } | |||
2452 | ||||
2453 | void CodeGenFunction::EmitOMPDistributeSimdDirective( | |||
2454 | const OMPDistributeSimdDirective &S) { | |||
2455 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2456 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
2457 | }; | |||
2458 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2459 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); | |||
2460 | } | |||
2461 | ||||
2462 | void CodeGenFunction::EmitOMPTargetSimdDeviceFunction( | |||
2463 | CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) { | |||
2464 | // Emit SPMD target parallel for region as a standalone region. | |||
2465 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
2466 | emitOMPSimdRegion(CGF, S, Action); | |||
2467 | }; | |||
2468 | llvm::Function *Fn; | |||
2469 | llvm::Constant *Addr; | |||
2470 | // Emit target region as a standalone region. | |||
2471 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
2472 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
2473 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 2473, __PRETTY_FUNCTION__)); | |||
2474 | } | |||
2475 | ||||
2476 | void CodeGenFunction::EmitOMPTargetSimdDirective( | |||
2477 | const OMPTargetSimdDirective &S) { | |||
2478 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
2479 | emitOMPSimdRegion(CGF, S, Action); | |||
2480 | }; | |||
2481 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
2482 | } | |||
2483 | ||||
2484 | namespace { | |||
2485 | struct ScheduleKindModifiersTy { | |||
2486 | OpenMPScheduleClauseKind Kind; | |||
2487 | OpenMPScheduleClauseModifier M1; | |||
2488 | OpenMPScheduleClauseModifier M2; | |||
2489 | ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind, | |||
2490 | OpenMPScheduleClauseModifier M1, | |||
2491 | OpenMPScheduleClauseModifier M2) | |||
2492 | : Kind(Kind), M1(M1), M2(M2) {} | |||
2493 | }; | |||
2494 | } // namespace | |||
2495 | ||||
2496 | bool CodeGenFunction::EmitOMPWorksharingLoop( | |||
2497 | const OMPLoopDirective &S, Expr *EUB, | |||
2498 | const CodeGenLoopBoundsTy &CodeGenLoopBounds, | |||
2499 | const CodeGenDispatchBoundsTy &CGDispatchBounds) { | |||
2500 | // Emit the loop iteration variable. | |||
2501 | const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); | |||
2502 | const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl()); | |||
2503 | EmitVarDecl(*IVDecl); | |||
2504 | ||||
2505 | // Emit the iterations count variable. | |||
2506 | // If it is not a variable, Sema decided to calculate iterations count on each | |||
2507 | // iteration (e.g., it is foldable into a constant). | |||
2508 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { | |||
2509 | EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); | |||
2510 | // Emit calculation of the iterations count. | |||
2511 | EmitIgnoredExpr(S.getCalcLastIteration()); | |||
2512 | } | |||
2513 | ||||
2514 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); | |||
2515 | ||||
2516 | bool HasLastprivateClause; | |||
2517 | // Check pre-condition. | |||
2518 | { | |||
2519 | OMPLoopScope PreInitScope(*this, S); | |||
2520 | // Skip the entire loop if we don't meet the precondition. | |||
2521 | // If the condition constant folds and can be elided, avoid emitting the | |||
2522 | // whole loop. | |||
2523 | bool CondConstant; | |||
2524 | llvm::BasicBlock *ContBlock = nullptr; | |||
2525 | if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { | |||
2526 | if (!CondConstant) | |||
2527 | return false; | |||
2528 | } else { | |||
2529 | llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); | |||
2530 | ContBlock = createBasicBlock("omp.precond.end"); | |||
2531 | emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, | |||
2532 | getProfileCount(&S)); | |||
2533 | EmitBlock(ThenBlock); | |||
2534 | incrementProfileCounter(&S); | |||
2535 | } | |||
2536 | ||||
2537 | RunCleanupsScope DoacrossCleanupScope(*this); | |||
2538 | bool Ordered = false; | |||
2539 | if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) { | |||
2540 | if (OrderedClause->getNumForLoops()) | |||
2541 | RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations()); | |||
2542 | else | |||
2543 | Ordered = true; | |||
2544 | } | |||
2545 | ||||
2546 | llvm::DenseSet<const Expr *> EmittedFinals; | |||
2547 | emitAlignedClause(*this, S); | |||
2548 | bool HasLinears = EmitOMPLinearClauseInit(S); | |||
2549 | // Emit helper vars inits. | |||
2550 | ||||
2551 | std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S); | |||
2552 | LValue LB = Bounds.first; | |||
2553 | LValue UB = Bounds.second; | |||
2554 | LValue ST = | |||
2555 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable())); | |||
2556 | LValue IL = | |||
2557 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); | |||
2558 | ||||
2559 | // Emit 'then' code. | |||
2560 | { | |||
2561 | OMPPrivateScope LoopScope(*this); | |||
2562 | if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) { | |||
2563 | // Emit implicit barrier to synchronize threads and avoid data races on | |||
2564 | // initialization of firstprivate variables and post-update of | |||
2565 | // lastprivate variables. | |||
2566 | CGM.getOpenMPRuntime().emitBarrierCall( | |||
2567 | *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, | |||
2568 | /*ForceSimpleCall=*/true); | |||
2569 | } | |||
2570 | EmitOMPPrivateClause(S, LoopScope); | |||
2571 | CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion( | |||
2572 | *this, S, EmitLValue(S.getIterationVariable())); | |||
2573 | HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); | |||
2574 | EmitOMPReductionClauseInit(S, LoopScope); | |||
2575 | EmitOMPPrivateLoopCounters(S, LoopScope); | |||
2576 | EmitOMPLinearClause(S, LoopScope); | |||
2577 | (void)LoopScope.Privatize(); | |||
2578 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
2579 | CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); | |||
2580 | ||||
2581 | // Detect the loop schedule kind and chunk. | |||
2582 | const Expr *ChunkExpr = nullptr; | |||
2583 | OpenMPScheduleTy ScheduleKind; | |||
2584 | if (const auto *C = S.getSingleClause<OMPScheduleClause>()) { | |||
2585 | ScheduleKind.Schedule = C->getScheduleKind(); | |||
2586 | ScheduleKind.M1 = C->getFirstScheduleModifier(); | |||
2587 | ScheduleKind.M2 = C->getSecondScheduleModifier(); | |||
2588 | ChunkExpr = C->getChunkSize(); | |||
2589 | } else { | |||
2590 | // Default behaviour for schedule clause. | |||
2591 | CGM.getOpenMPRuntime().getDefaultScheduleAndChunk( | |||
2592 | *this, S, ScheduleKind.Schedule, ChunkExpr); | |||
2593 | } | |||
2594 | bool HasChunkSizeOne = false; | |||
2595 | llvm::Value *Chunk = nullptr; | |||
2596 | if (ChunkExpr) { | |||
2597 | Chunk = EmitScalarExpr(ChunkExpr); | |||
2598 | Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(), | |||
2599 | S.getIterationVariable()->getType(), | |||
2600 | S.getBeginLoc()); | |||
2601 | Expr::EvalResult Result; | |||
2602 | if (ChunkExpr->EvaluateAsInt(Result, getContext())) { | |||
2603 | llvm::APSInt EvaluatedChunk = Result.Val.getInt(); | |||
2604 | HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1); | |||
2605 | } | |||
2606 | } | |||
2607 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); | |||
2608 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); | |||
2609 | // OpenMP 4.5, 2.7.1 Loop Construct, Description. | |||
2610 | // If the static schedule kind is specified or if the ordered clause is | |||
2611 | // specified, and if no monotonic modifier is specified, the effect will | |||
2612 | // be as if the monotonic modifier was specified. | |||
2613 | bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule, | |||
2614 | /* Chunked */ Chunk != nullptr) && HasChunkSizeOne && | |||
2615 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()); | |||
2616 | if ((RT.isStaticNonchunked(ScheduleKind.Schedule, | |||
2617 | /* Chunked */ Chunk != nullptr) || | |||
2618 | StaticChunkedOne) && | |||
2619 | !Ordered) { | |||
2620 | JumpDest LoopExit = | |||
2621 | getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); | |||
2622 | emitCommonSimdLoop( | |||
2623 | *this, S, | |||
2624 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2625 | if (isOpenMPSimdDirective(S.getDirectiveKind())) | |||
2626 | CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); | |||
2627 | }, | |||
2628 | [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk, | |||
2629 | &S, ScheduleKind, LoopExit, | |||
2630 | &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2631 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] | |||
2632 | // When no chunk_size is specified, the iteration space is divided | |||
2633 | // into chunks that are approximately equal in size, and at most | |||
2634 | // one chunk is distributed to each thread. Note that the size of | |||
2635 | // the chunks is unspecified in this case. | |||
2636 | CGOpenMPRuntime::StaticRTInput StaticInit( | |||
2637 | IVSize, IVSigned, Ordered, IL.getAddress(CGF), | |||
2638 | LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), | |||
2639 | StaticChunkedOne ? Chunk : nullptr); | |||
2640 | CGF.CGM.getOpenMPRuntime().emitForStaticInit( | |||
2641 | CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, | |||
2642 | StaticInit); | |||
2643 | // UB = min(UB, GlobalUB); | |||
2644 | if (!StaticChunkedOne) | |||
2645 | CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); | |||
2646 | // IV = LB; | |||
2647 | CGF.EmitIgnoredExpr(S.getInit()); | |||
2648 | // For unchunked static schedule generate: | |||
2649 | // | |||
2650 | // while (idx <= UB) { | |||
2651 | // BODY; | |||
2652 | // ++idx; | |||
2653 | // } | |||
2654 | // | |||
2655 | // For static schedule with chunk one: | |||
2656 | // | |||
2657 | // while (IV <= PrevUB) { | |||
2658 | // BODY; | |||
2659 | // IV += ST; | |||
2660 | // } | |||
2661 | CGF.EmitOMPInnerLoop( | |||
2662 | S, LoopScope.requiresCleanups(), | |||
2663 | StaticChunkedOne ? S.getCombinedParForInDistCond() | |||
2664 | : S.getCond(), | |||
2665 | StaticChunkedOne ? S.getDistInc() : S.getInc(), | |||
2666 | [&S, LoopExit](CodeGenFunction &CGF) { | |||
2667 | CGF.CGM.getOpenMPRuntime() | |||
2668 | .initLastprivateConditionalCounter(CGF, S); | |||
2669 | CGF.EmitOMPLoopBody(S, LoopExit); | |||
2670 | CGF.EmitStopPoint(&S); | |||
2671 | }, | |||
2672 | [](CodeGenFunction &) {}); | |||
2673 | }); | |||
2674 | EmitBlock(LoopExit.getBlock()); | |||
2675 | // Tell the runtime we are done. | |||
2676 | auto &&CodeGen = [&S](CodeGenFunction &CGF) { | |||
2677 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), | |||
2678 | S.getDirectiveKind()); | |||
2679 | }; | |||
2680 | OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); | |||
2681 | } else { | |||
2682 | const bool IsMonotonic = | |||
2683 | Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static || | |||
2684 | ScheduleKind.Schedule == OMPC_SCHEDULE_unknown || | |||
2685 | ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic || | |||
2686 | ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic; | |||
2687 | // Emit the outer loop, which requests its work chunk [LB..UB] from | |||
2688 | // runtime and runs the inner loop to process it. | |||
2689 | const OMPLoopArguments LoopArguments( | |||
2690 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), | |||
2691 | IL.getAddress(*this), Chunk, EUB); | |||
2692 | EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered, | |||
2693 | LoopArguments, CGDispatchBounds); | |||
2694 | } | |||
2695 | if (isOpenMPSimdDirective(S.getDirectiveKind())) { | |||
2696 | EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { | |||
2697 | return CGF.Builder.CreateIsNotNull( | |||
2698 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
2699 | }); | |||
2700 | } | |||
2701 | EmitOMPReductionClauseFinal( | |||
2702 | S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind()) | |||
2703 | ? /*Parallel and Simd*/ OMPD_parallel_for_simd | |||
2704 | : /*Parallel only*/ OMPD_parallel); | |||
2705 | // Emit post-update of the reduction variables if IsLastIter != 0. | |||
2706 | emitPostUpdateForReductionClause( | |||
2707 | *this, S, [IL, &S](CodeGenFunction &CGF) { | |||
2708 | return CGF.Builder.CreateIsNotNull( | |||
2709 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
2710 | }); | |||
2711 | // Emit final copy of the lastprivate variables if IsLastIter != 0. | |||
2712 | if (HasLastprivateClause) | |||
2713 | EmitOMPLastprivateClauseFinal( | |||
2714 | S, isOpenMPSimdDirective(S.getDirectiveKind()), | |||
2715 | Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); | |||
2716 | } | |||
2717 | EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { | |||
2718 | return CGF.Builder.CreateIsNotNull( | |||
2719 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
2720 | }); | |||
2721 | DoacrossCleanupScope.ForceCleanup(); | |||
2722 | // We're now done with the loop, so jump to the continuation block. | |||
2723 | if (ContBlock) { | |||
2724 | EmitBranch(ContBlock); | |||
2725 | EmitBlock(ContBlock, /*IsFinished=*/true); | |||
2726 | } | |||
2727 | } | |||
2728 | return HasLastprivateClause; | |||
2729 | } | |||
2730 | ||||
2731 | /// The following two functions generate expressions for the loop lower | |||
2732 | /// and upper bounds in case of static and dynamic (dispatch) schedule | |||
2733 | /// of the associated 'for' or 'distribute' loop. | |||
2734 | static std::pair<LValue, LValue> | |||
2735 | emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { | |||
2736 | const auto &LS = cast<OMPLoopDirective>(S); | |||
2737 | LValue LB = | |||
2738 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); | |||
2739 | LValue UB = | |||
2740 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); | |||
2741 | return {LB, UB}; | |||
2742 | } | |||
2743 | ||||
2744 | /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not | |||
2745 | /// consider the lower and upper bound expressions generated by the | |||
2746 | /// worksharing loop support, but we use 0 and the iteration space size as | |||
2747 | /// constants | |||
2748 | static std::pair<llvm::Value *, llvm::Value *> | |||
2749 | emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, | |||
2750 | Address LB, Address UB) { | |||
2751 | const auto &LS = cast<OMPLoopDirective>(S); | |||
2752 | const Expr *IVExpr = LS.getIterationVariable(); | |||
2753 | const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); | |||
2754 | llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); | |||
2755 | llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); | |||
2756 | return {LBVal, UBVal}; | |||
2757 | } | |||
2758 | ||||
2759 | void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { | |||
2760 | bool HasLastprivates = false; | |||
2761 | auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, | |||
2762 | PrePostActionTy &) { | |||
2763 | OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel()); | |||
2764 | HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), | |||
2765 | emitForLoopBounds, | |||
2766 | emitDispatchForLoopBounds); | |||
2767 | }; | |||
2768 | { | |||
2769 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2770 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, | |||
2771 | S.hasCancel()); | |||
2772 | } | |||
2773 | ||||
2774 | // Emit an implicit barrier at the end. | |||
2775 | if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) | |||
2776 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for); | |||
2777 | } | |||
2778 | ||||
2779 | void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { | |||
2780 | bool HasLastprivates = false; | |||
2781 | auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, | |||
2782 | PrePostActionTy &) { | |||
2783 | HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), | |||
2784 | emitForLoopBounds, | |||
2785 | emitDispatchForLoopBounds); | |||
2786 | }; | |||
2787 | { | |||
2788 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2789 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); | |||
2790 | } | |||
2791 | ||||
2792 | // Emit an implicit barrier at the end. | |||
2793 | if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) | |||
2794 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for); | |||
2795 | } | |||
2796 | ||||
2797 | static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, | |||
2798 | const Twine &Name, | |||
2799 | llvm::Value *Init = nullptr) { | |||
2800 | LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); | |||
2801 | if (Init) | |||
2802 | CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); | |||
2803 | return LVal; | |||
2804 | } | |||
2805 | ||||
2806 | void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { | |||
2807 | const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt(); | |||
2808 | const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt); | |||
2809 | bool HasLastprivates = false; | |||
2810 | auto &&CodeGen = [&S, CapturedStmt, CS, | |||
2811 | &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2812 | ASTContext &C = CGF.getContext(); | |||
2813 | QualType KmpInt32Ty = | |||
2814 | C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); | |||
2815 | // Emit helper vars inits. | |||
2816 | LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", | |||
2817 | CGF.Builder.getInt32(0)); | |||
2818 | llvm::ConstantInt *GlobalUBVal = CS != nullptr | |||
2819 | ? CGF.Builder.getInt32(CS->size() - 1) | |||
2820 | : CGF.Builder.getInt32(0); | |||
2821 | LValue UB = | |||
2822 | createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); | |||
2823 | LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", | |||
2824 | CGF.Builder.getInt32(1)); | |||
2825 | LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", | |||
2826 | CGF.Builder.getInt32(0)); | |||
2827 | // Loop counter. | |||
2828 | LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); | |||
2829 | OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue); | |||
2830 | CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); | |||
2831 | OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue); | |||
2832 | CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); | |||
2833 | // Generate condition for loop. | |||
2834 | BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, | |||
2835 | OK_Ordinary, S.getBeginLoc(), FPOptions()); | |||
2836 | // Increment for loop counter. | |||
2837 | UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary, | |||
2838 | S.getBeginLoc(), true); | |||
2839 | auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { | |||
2840 | // Iterate through all sections and emit a switch construct: | |||
2841 | // switch (IV) { | |||
2842 | // case 0: | |||
2843 | // <SectionStmt[0]>; | |||
2844 | // break; | |||
2845 | // ... | |||
2846 | // case <NumSection> - 1: | |||
2847 | // <SectionStmt[<NumSection> - 1]>; | |||
2848 | // break; | |||
2849 | // } | |||
2850 | // .omp.sections.exit: | |||
2851 | CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(CGF, S); | |||
2852 | llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); | |||
2853 | llvm::SwitchInst *SwitchStmt = | |||
2854 | CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), | |||
2855 | ExitBB, CS == nullptr ? 1 : CS->size()); | |||
2856 | if (CS) { | |||
2857 | unsigned CaseNumber = 0; | |||
2858 | for (const Stmt *SubStmt : CS->children()) { | |||
2859 | auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); | |||
2860 | CGF.EmitBlock(CaseBB); | |||
2861 | SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); | |||
2862 | CGF.EmitStmt(SubStmt); | |||
2863 | CGF.EmitBranch(ExitBB); | |||
2864 | ++CaseNumber; | |||
2865 | } | |||
2866 | } else { | |||
2867 | llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); | |||
2868 | CGF.EmitBlock(CaseBB); | |||
2869 | SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); | |||
2870 | CGF.EmitStmt(CapturedStmt); | |||
2871 | CGF.EmitBranch(ExitBB); | |||
2872 | } | |||
2873 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); | |||
2874 | }; | |||
2875 | ||||
2876 | CodeGenFunction::OMPPrivateScope LoopScope(CGF); | |||
2877 | if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { | |||
2878 | // Emit implicit barrier to synchronize threads and avoid data races on | |||
2879 | // initialization of firstprivate variables and post-update of lastprivate | |||
2880 | // variables. | |||
2881 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( | |||
2882 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, | |||
2883 | /*ForceSimpleCall=*/true); | |||
2884 | } | |||
2885 | CGF.EmitOMPPrivateClause(S, LoopScope); | |||
2886 | CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); | |||
2887 | HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); | |||
2888 | CGF.EmitOMPReductionClauseInit(S, LoopScope); | |||
2889 | (void)LoopScope.Privatize(); | |||
2890 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
2891 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); | |||
2892 | ||||
2893 | // Emit static non-chunked loop. | |||
2894 | OpenMPScheduleTy ScheduleKind; | |||
2895 | ScheduleKind.Schedule = OMPC_SCHEDULE_static; | |||
2896 | CGOpenMPRuntime::StaticRTInput StaticInit( | |||
2897 | /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), | |||
2898 | LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); | |||
2899 | CGF.CGM.getOpenMPRuntime().emitForStaticInit( | |||
2900 | CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); | |||
2901 | // UB = min(UB, GlobalUB); | |||
2902 | llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); | |||
2903 | llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( | |||
2904 | CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); | |||
2905 | CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); | |||
2906 | // IV = LB; | |||
2907 | CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); | |||
2908 | // while (idx <= UB) { BODY; ++idx; } | |||
2909 | CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, | |||
2910 | [](CodeGenFunction &) {}); | |||
2911 | // Tell the runtime we are done. | |||
2912 | auto &&CodeGen = [&S](CodeGenFunction &CGF) { | |||
2913 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), | |||
2914 | S.getDirectiveKind()); | |||
2915 | }; | |||
2916 | CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); | |||
2917 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); | |||
2918 | // Emit post-update of the reduction variables if IsLastIter != 0. | |||
2919 | emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { | |||
2920 | return CGF.Builder.CreateIsNotNull( | |||
2921 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
2922 | }); | |||
2923 | ||||
2924 | // Emit final copy of the lastprivate variables if IsLastIter != 0. | |||
2925 | if (HasLastprivates) | |||
2926 | CGF.EmitOMPLastprivateClauseFinal( | |||
2927 | S, /*NoFinals=*/false, | |||
2928 | CGF.Builder.CreateIsNotNull( | |||
2929 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); | |||
2930 | }; | |||
2931 | ||||
2932 | bool HasCancel = false; | |||
2933 | if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S)) | |||
2934 | HasCancel = OSD->hasCancel(); | |||
2935 | else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S)) | |||
2936 | HasCancel = OPSD->hasCancel(); | |||
2937 | OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel); | |||
2938 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen, | |||
2939 | HasCancel); | |||
2940 | // Emit barrier for lastprivates only if 'sections' directive has 'nowait' | |||
2941 | // clause. Otherwise the barrier will be generated by the codegen for the | |||
2942 | // directive. | |||
2943 | if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) { | |||
2944 | // Emit implicit barrier to synchronize threads and avoid data races on | |||
2945 | // initialization of firstprivate variables. | |||
2946 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), | |||
2947 | OMPD_unknown); | |||
2948 | } | |||
2949 | } | |||
2950 | ||||
2951 | void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { | |||
2952 | { | |||
2953 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2954 | EmitSections(S); | |||
2955 | } | |||
2956 | // Emit an implicit barrier at the end. | |||
2957 | if (!S.getSingleClause<OMPNowaitClause>()) { | |||
2958 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), | |||
2959 | OMPD_sections); | |||
2960 | } | |||
2961 | } | |||
2962 | ||||
2963 | void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { | |||
2964 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
2965 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
2966 | }; | |||
2967 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
2968 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen, | |||
2969 | S.hasCancel()); | |||
2970 | } | |||
2971 | ||||
2972 | void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { | |||
2973 | llvm::SmallVector<const Expr *, 8> CopyprivateVars; | |||
2974 | llvm::SmallVector<const Expr *, 8> DestExprs; | |||
2975 | llvm::SmallVector<const Expr *, 8> SrcExprs; | |||
2976 | llvm::SmallVector<const Expr *, 8> AssignmentOps; | |||
2977 | // Check if there are any 'copyprivate' clauses associated with this | |||
2978 | // 'single' construct. | |||
2979 | // Build a list of copyprivate variables along with helper expressions | |||
2980 | // (<source>, <destination>, <destination>=<source> expressions) | |||
2981 | for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) { | |||
2982 | CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); | |||
2983 | DestExprs.append(C->destination_exprs().begin(), | |||
2984 | C->destination_exprs().end()); | |||
2985 | SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); | |||
2986 | AssignmentOps.append(C->assignment_ops().begin(), | |||
2987 | C->assignment_ops().end()); | |||
2988 | } | |||
2989 | // Emit code for 'single' region along with 'copyprivate' clauses | |||
2990 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
2991 | Action.Enter(CGF); | |||
2992 | OMPPrivateScope SingleScope(CGF); | |||
2993 | (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); | |||
2994 | CGF.EmitOMPPrivateClause(S, SingleScope); | |||
2995 | (void)SingleScope.Privatize(); | |||
2996 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
2997 | }; | |||
2998 | { | |||
2999 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3000 | CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(), | |||
3001 | CopyprivateVars, DestExprs, | |||
3002 | SrcExprs, AssignmentOps); | |||
3003 | } | |||
3004 | // Emit an implicit barrier at the end (to avoid data race on firstprivate | |||
3005 | // init or if no 'nowait' clause was specified and no 'copyprivate' clause). | |||
3006 | if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) { | |||
3007 | CGM.getOpenMPRuntime().emitBarrierCall( | |||
3008 | *this, S.getBeginLoc(), | |||
3009 | S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single); | |||
3010 | } | |||
3011 | } | |||
3012 | ||||
3013 | static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { | |||
3014 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3015 | Action.Enter(CGF); | |||
3016 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
3017 | }; | |||
3018 | CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); | |||
3019 | } | |||
3020 | ||||
3021 | void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { | |||
3022 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3023 | emitMaster(*this, S); | |||
3024 | } | |||
3025 | ||||
3026 | void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { | |||
3027 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3028 | Action.Enter(CGF); | |||
3029 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
3030 | }; | |||
3031 | const Expr *Hint = nullptr; | |||
3032 | if (const auto *HintClause = S.getSingleClause<OMPHintClause>()) | |||
3033 | Hint = HintClause->getHint(); | |||
3034 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3035 | CGM.getOpenMPRuntime().emitCriticalRegion(*this, | |||
3036 | S.getDirectiveName().getAsString(), | |||
3037 | CodeGen, S.getBeginLoc(), Hint); | |||
3038 | } | |||
3039 | ||||
3040 | void CodeGenFunction::EmitOMPParallelForDirective( | |||
3041 | const OMPParallelForDirective &S) { | |||
3042 | // Emit directive as a combined directive that consists of two implicit | |||
3043 | // directives: 'parallel' with 'for' directive. | |||
3044 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3045 | Action.Enter(CGF); | |||
3046 | OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel()); | |||
3047 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, | |||
3048 | emitDispatchForLoopBounds); | |||
3049 | }; | |||
3050 | emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen, | |||
3051 | emitEmptyBoundParameters); | |||
3052 | } | |||
3053 | ||||
3054 | void CodeGenFunction::EmitOMPParallelForSimdDirective( | |||
3055 | const OMPParallelForSimdDirective &S) { | |||
3056 | // Emit directive as a combined directive that consists of two implicit | |||
3057 | // directives: 'parallel' with 'for' directive. | |||
3058 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3059 | Action.Enter(CGF); | |||
3060 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, | |||
3061 | emitDispatchForLoopBounds); | |||
3062 | }; | |||
3063 | emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen, | |||
3064 | emitEmptyBoundParameters); | |||
3065 | } | |||
3066 | ||||
3067 | void CodeGenFunction::EmitOMPParallelMasterDirective( | |||
3068 | const OMPParallelMasterDirective &S) { | |||
3069 | // Emit directive as a combined directive that consists of two implicit | |||
3070 | // directives: 'parallel' with 'master' directive. | |||
3071 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3072 | Action.Enter(CGF); | |||
3073 | OMPPrivateScope PrivateScope(CGF); | |||
3074 | bool Copyins = CGF.EmitOMPCopyinClause(S); | |||
3075 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
3076 | if (Copyins) { | |||
3077 | // Emit implicit barrier to synchronize threads and avoid data races on | |||
3078 | // propagation master's thread values of threadprivate variables to local | |||
3079 | // instances of that variables of all other implicit threads. | |||
3080 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( | |||
3081 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, | |||
3082 | /*ForceSimpleCall=*/true); | |||
3083 | } | |||
3084 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
3085 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
3086 | (void)PrivateScope.Privatize(); | |||
3087 | emitMaster(CGF, S); | |||
3088 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); | |||
3089 | }; | |||
3090 | emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen, | |||
3091 | emitEmptyBoundParameters); | |||
3092 | emitPostUpdateForReductionClause(*this, S, | |||
3093 | [](CodeGenFunction &) { return nullptr; }); | |||
3094 | } | |||
3095 | ||||
3096 | void CodeGenFunction::EmitOMPParallelSectionsDirective( | |||
3097 | const OMPParallelSectionsDirective &S) { | |||
3098 | // Emit directive as a combined directive that consists of two implicit | |||
3099 | // directives: 'parallel' with 'sections' directive. | |||
3100 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3101 | Action.Enter(CGF); | |||
3102 | CGF.EmitSections(S); | |||
3103 | }; | |||
3104 | emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen, | |||
3105 | emitEmptyBoundParameters); | |||
3106 | } | |||
3107 | ||||
3108 | void CodeGenFunction::EmitOMPTaskBasedDirective( | |||
3109 | const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, | |||
3110 | const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, | |||
3111 | OMPTaskDataTy &Data) { | |||
3112 | // Emit outlined function for task construct. | |||
3113 | const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion); | |||
3114 | auto I = CS->getCapturedDecl()->param_begin(); | |||
3115 | auto PartId = std::next(I); | |||
3116 | auto TaskT = std::next(I, 4); | |||
3117 | // Check if the task is final | |||
3118 | if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) { | |||
3119 | // If the condition constant folds and can be elided, try to avoid emitting | |||
3120 | // the condition and the dead arm of the if/else. | |||
3121 | const Expr *Cond = Clause->getCondition(); | |||
3122 | bool CondConstant; | |||
3123 | if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) | |||
3124 | Data.Final.setInt(CondConstant); | |||
3125 | else | |||
3126 | Data.Final.setPointer(EvaluateExprAsBool(Cond)); | |||
3127 | } else { | |||
3128 | // By default the task is not final. | |||
3129 | Data.Final.setInt(/*IntVal=*/false); | |||
3130 | } | |||
3131 | // Check if the task has 'priority' clause. | |||
3132 | if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) { | |||
3133 | const Expr *Prio = Clause->getPriority(); | |||
3134 | Data.Priority.setInt(/*IntVal=*/true); | |||
3135 | Data.Priority.setPointer(EmitScalarConversion( | |||
3136 | EmitScalarExpr(Prio), Prio->getType(), | |||
3137 | getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), | |||
3138 | Prio->getExprLoc())); | |||
3139 | } | |||
3140 | // The first function argument for tasks is a thread id, the second one is a | |||
3141 | // part id (0 for tied tasks, >=0 for untied task). | |||
3142 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; | |||
3143 | // Get list of private variables. | |||
3144 | for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { | |||
3145 | auto IRef = C->varlist_begin(); | |||
3146 | for (const Expr *IInit : C->private_copies()) { | |||
3147 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
3148 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
3149 | Data.PrivateVars.push_back(*IRef); | |||
3150 | Data.PrivateCopies.push_back(IInit); | |||
3151 | } | |||
3152 | ++IRef; | |||
3153 | } | |||
3154 | } | |||
3155 | EmittedAsPrivate.clear(); | |||
3156 | // Get list of firstprivate variables. | |||
3157 | for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { | |||
3158 | auto IRef = C->varlist_begin(); | |||
3159 | auto IElemInitRef = C->inits().begin(); | |||
3160 | for (const Expr *IInit : C->private_copies()) { | |||
3161 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
3162 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
3163 | Data.FirstprivateVars.push_back(*IRef); | |||
3164 | Data.FirstprivateCopies.push_back(IInit); | |||
3165 | Data.FirstprivateInits.push_back(*IElemInitRef); | |||
3166 | } | |||
3167 | ++IRef; | |||
3168 | ++IElemInitRef; | |||
3169 | } | |||
3170 | } | |||
3171 | // Get list of lastprivate variables (for taskloops). | |||
3172 | llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs; | |||
3173 | for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) { | |||
3174 | auto IRef = C->varlist_begin(); | |||
3175 | auto ID = C->destination_exprs().begin(); | |||
3176 | for (const Expr *IInit : C->private_copies()) { | |||
3177 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); | |||
3178 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { | |||
3179 | Data.LastprivateVars.push_back(*IRef); | |||
3180 | Data.LastprivateCopies.push_back(IInit); | |||
3181 | } | |||
3182 | LastprivateDstsOrigs.insert( | |||
3183 | {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()), | |||
3184 | cast<DeclRefExpr>(*IRef)}); | |||
3185 | ++IRef; | |||
3186 | ++ID; | |||
3187 | } | |||
3188 | } | |||
3189 | SmallVector<const Expr *, 4> LHSs; | |||
3190 | SmallVector<const Expr *, 4> RHSs; | |||
3191 | for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) { | |||
3192 | auto IPriv = C->privates().begin(); | |||
3193 | auto IRed = C->reduction_ops().begin(); | |||
3194 | auto ILHS = C->lhs_exprs().begin(); | |||
3195 | auto IRHS = C->rhs_exprs().begin(); | |||
3196 | for (const Expr *Ref : C->varlists()) { | |||
3197 | Data.ReductionVars.emplace_back(Ref); | |||
3198 | Data.ReductionCopies.emplace_back(*IPriv); | |||
3199 | Data.ReductionOps.emplace_back(*IRed); | |||
3200 | LHSs.emplace_back(*ILHS); | |||
3201 | RHSs.emplace_back(*IRHS); | |||
3202 | std::advance(IPriv, 1); | |||
3203 | std::advance(IRed, 1); | |||
3204 | std::advance(ILHS, 1); | |||
3205 | std::advance(IRHS, 1); | |||
3206 | } | |||
3207 | } | |||
3208 | Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit( | |||
3209 | *this, S.getBeginLoc(), LHSs, RHSs, Data); | |||
3210 | // Build list of dependences. | |||
3211 | for (const auto *C : S.getClausesOfKind<OMPDependClause>()) | |||
3212 | for (const Expr *IRef : C->varlists()) | |||
3213 | Data.Dependences.emplace_back(C->getDependencyKind(), IRef); | |||
3214 | auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs, | |||
3215 | CapturedRegion](CodeGenFunction &CGF, | |||
3216 | PrePostActionTy &Action) { | |||
3217 | // Set proper addresses for generated private copies. | |||
3218 | OMPPrivateScope Scope(CGF); | |||
3219 | if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() || | |||
3220 | !Data.LastprivateVars.empty()) { | |||
3221 | llvm::FunctionType *CopyFnTy = llvm::FunctionType::get( | |||
3222 | CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); | |||
3223 | enum { PrivatesParam = 2, CopyFnParam = 3 }; | |||
3224 | llvm::Value *CopyFn = CGF.Builder.CreateLoad( | |||
3225 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); | |||
3226 | llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( | |||
3227 | CS->getCapturedDecl()->getParam(PrivatesParam))); | |||
3228 | // Map privates. | |||
3229 | llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs; | |||
3230 | llvm::SmallVector<llvm::Value *, 16> CallArgs; | |||
3231 | CallArgs.push_back(PrivatesPtr); | |||
3232 | for (const Expr *E : Data.PrivateVars) { | |||
3233 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
3234 | Address PrivatePtr = CGF.CreateMemTemp( | |||
3235 | CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); | |||
3236 | PrivatePtrs.emplace_back(VD, PrivatePtr); | |||
3237 | CallArgs.push_back(PrivatePtr.getPointer()); | |||
3238 | } | |||
3239 | for (const Expr *E : Data.FirstprivateVars) { | |||
3240 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
3241 | Address PrivatePtr = | |||
3242 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), | |||
3243 | ".firstpriv.ptr.addr"); | |||
3244 | PrivatePtrs.emplace_back(VD, PrivatePtr); | |||
3245 | CallArgs.push_back(PrivatePtr.getPointer()); | |||
3246 | } | |||
3247 | for (const Expr *E : Data.LastprivateVars) { | |||
3248 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
3249 | Address PrivatePtr = | |||
3250 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), | |||
3251 | ".lastpriv.ptr.addr"); | |||
3252 | PrivatePtrs.emplace_back(VD, PrivatePtr); | |||
3253 | CallArgs.push_back(PrivatePtr.getPointer()); | |||
3254 | } | |||
3255 | CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( | |||
3256 | CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); | |||
3257 | for (const auto &Pair : LastprivateDstsOrigs) { | |||
3258 | const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl()); | |||
3259 | DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), | |||
3260 | /*RefersToEnclosingVariableOrCapture=*/ | |||
3261 | CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, | |||
3262 | Pair.second->getType(), VK_LValue, | |||
3263 | Pair.second->getExprLoc()); | |||
3264 | Scope.addPrivate(Pair.first, [&CGF, &DRE]() { | |||
3265 | return CGF.EmitLValue(&DRE).getAddress(CGF); | |||
3266 | }); | |||
3267 | } | |||
3268 | for (const auto &Pair : PrivatePtrs) { | |||
3269 | Address Replacement(CGF.Builder.CreateLoad(Pair.second), | |||
3270 | CGF.getContext().getDeclAlign(Pair.first)); | |||
3271 | Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); | |||
3272 | } | |||
3273 | } | |||
3274 | if (Data.Reductions) { | |||
3275 | OMPLexicalScope LexScope(CGF, S, CapturedRegion); | |||
3276 | ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies, | |||
3277 | Data.ReductionOps); | |||
3278 | llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( | |||
3279 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); | |||
3280 | for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) { | |||
3281 | RedCG.emitSharedLValue(CGF, Cnt); | |||
3282 | RedCG.emitAggregateType(CGF, Cnt); | |||
3283 | // FIXME: This must removed once the runtime library is fixed. | |||
3284 | // Emit required threadprivate variables for | |||
3285 | // initializer/combiner/finalizer. | |||
3286 | CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), | |||
3287 | RedCG, Cnt); | |||
3288 | Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( | |||
3289 | CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); | |||
3290 | Replacement = | |||
3291 | Address(CGF.EmitScalarConversion( | |||
3292 | Replacement.getPointer(), CGF.getContext().VoidPtrTy, | |||
3293 | CGF.getContext().getPointerType( | |||
3294 | Data.ReductionCopies[Cnt]->getType()), | |||
3295 | Data.ReductionCopies[Cnt]->getExprLoc()), | |||
3296 | Replacement.getAlignment()); | |||
3297 | Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); | |||
3298 | Scope.addPrivate(RedCG.getBaseDecl(Cnt), | |||
3299 | [Replacement]() { return Replacement; }); | |||
3300 | } | |||
3301 | } | |||
3302 | // Privatize all private variables except for in_reduction items. | |||
3303 | (void)Scope.Privatize(); | |||
3304 | SmallVector<const Expr *, 4> InRedVars; | |||
3305 | SmallVector<const Expr *, 4> InRedPrivs; | |||
3306 | SmallVector<const Expr *, 4> InRedOps; | |||
3307 | SmallVector<const Expr *, 4> TaskgroupDescriptors; | |||
3308 | for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) { | |||
3309 | auto IPriv = C->privates().begin(); | |||
3310 | auto IRed = C->reduction_ops().begin(); | |||
3311 | auto ITD = C->taskgroup_descriptors().begin(); | |||
3312 | for (const Expr *Ref : C->varlists()) { | |||
3313 | InRedVars.emplace_back(Ref); | |||
3314 | InRedPrivs.emplace_back(*IPriv); | |||
3315 | InRedOps.emplace_back(*IRed); | |||
3316 | TaskgroupDescriptors.emplace_back(*ITD); | |||
3317 | std::advance(IPriv, 1); | |||
3318 | std::advance(IRed, 1); | |||
3319 | std::advance(ITD, 1); | |||
3320 | } | |||
3321 | } | |||
3322 | // Privatize in_reduction items here, because taskgroup descriptors must be | |||
3323 | // privatized earlier. | |||
3324 | OMPPrivateScope InRedScope(CGF); | |||
3325 | if (!InRedVars.empty()) { | |||
3326 | ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps); | |||
3327 | for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) { | |||
3328 | RedCG.emitSharedLValue(CGF, Cnt); | |||
3329 | RedCG.emitAggregateType(CGF, Cnt); | |||
3330 | // The taskgroup descriptor variable is always implicit firstprivate and | |||
3331 | // privatized already during processing of the firstprivates. | |||
3332 | // FIXME: This must removed once the runtime library is fixed. | |||
3333 | // Emit required threadprivate variables for | |||
3334 | // initializer/combiner/finalizer. | |||
3335 | CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), | |||
3336 | RedCG, Cnt); | |||
3337 | llvm::Value *ReductionsPtr = | |||
3338 | CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), | |||
3339 | TaskgroupDescriptors[Cnt]->getExprLoc()); | |||
3340 | Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( | |||
3341 | CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); | |||
3342 | Replacement = Address( | |||
3343 | CGF.EmitScalarConversion( | |||
3344 | Replacement.getPointer(), CGF.getContext().VoidPtrTy, | |||
3345 | CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), | |||
3346 | InRedPrivs[Cnt]->getExprLoc()), | |||
3347 | Replacement.getAlignment()); | |||
3348 | Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); | |||
3349 | InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), | |||
3350 | [Replacement]() { return Replacement; }); | |||
3351 | } | |||
3352 | } | |||
3353 | (void)InRedScope.Privatize(); | |||
3354 | ||||
3355 | Action.Enter(CGF); | |||
3356 | BodyGen(CGF); | |||
3357 | }; | |||
3358 | llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( | |||
3359 | S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied, | |||
3360 | Data.NumberOfParts); | |||
3361 | OMPLexicalScope Scope(*this, S, llvm::None, | |||
3362 | !isOpenMPParallelDirective(S.getDirectiveKind()) && | |||
3363 | !isOpenMPSimdDirective(S.getDirectiveKind())); | |||
3364 | TaskGen(*this, OutlinedFn, Data); | |||
3365 | } | |||
3366 | ||||
3367 | static ImplicitParamDecl * | |||
3368 | createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, | |||
3369 | QualType Ty, CapturedDecl *CD, | |||
3370 | SourceLocation Loc) { | |||
3371 | auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, | |||
3372 | ImplicitParamDecl::Other); | |||
3373 | auto *OrigRef = DeclRefExpr::Create( | |||
3374 | C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD, | |||
3375 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); | |||
3376 | auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, | |||
3377 | ImplicitParamDecl::Other); | |||
3378 | auto *PrivateRef = DeclRefExpr::Create( | |||
3379 | C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD, | |||
3380 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); | |||
3381 | QualType ElemType = C.getBaseElementType(Ty); | |||
3382 | auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType, | |||
3383 | ImplicitParamDecl::Other); | |||
3384 | auto *InitRef = DeclRefExpr::Create( | |||
3385 | C, NestedNameSpecifierLoc(), SourceLocation(), InitVD, | |||
3386 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue); | |||
3387 | PrivateVD->setInitStyle(VarDecl::CInit); | |||
3388 | PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue, | |||
3389 | InitRef, /*BasePath=*/nullptr, | |||
3390 | VK_RValue)); | |||
3391 | Data.FirstprivateVars.emplace_back(OrigRef); | |||
3392 | Data.FirstprivateCopies.emplace_back(PrivateRef); | |||
3393 | Data.FirstprivateInits.emplace_back(InitRef); | |||
3394 | return OrigVD; | |||
3395 | } | |||
3396 | ||||
3397 | void CodeGenFunction::EmitOMPTargetTaskBasedDirective( | |||
3398 | const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, | |||
3399 | OMPTargetDataInfo &InputInfo) { | |||
3400 | // Emit outlined function for task construct. | |||
3401 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); | |||
3402 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); | |||
3403 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); | |||
3404 | auto I = CS->getCapturedDecl()->param_begin(); | |||
3405 | auto PartId = std::next(I); | |||
3406 | auto TaskT = std::next(I, 4); | |||
3407 | OMPTaskDataTy Data; | |||
3408 | // The task is not final. | |||
3409 | Data.Final.setInt(/*IntVal=*/false); | |||
3410 | // Get list of firstprivate variables. | |||
3411 | for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { | |||
3412 | auto IRef = C->varlist_begin(); | |||
3413 | auto IElemInitRef = C->inits().begin(); | |||
3414 | for (auto *IInit : C->private_copies()) { | |||
3415 | Data.FirstprivateVars.push_back(*IRef); | |||
3416 | Data.FirstprivateCopies.push_back(IInit); | |||
3417 | Data.FirstprivateInits.push_back(*IElemInitRef); | |||
3418 | ++IRef; | |||
3419 | ++IElemInitRef; | |||
3420 | } | |||
3421 | } | |||
3422 | OMPPrivateScope TargetScope(*this); | |||
3423 | VarDecl *BPVD = nullptr; | |||
3424 | VarDecl *PVD = nullptr; | |||
3425 | VarDecl *SVD = nullptr; | |||
3426 | if (InputInfo.NumberOfTargetItems > 0) { | |||
3427 | auto *CD = CapturedDecl::Create( | |||
3428 | getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0); | |||
3429 | llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems); | |||
3430 | QualType BaseAndPointersType = getContext().getConstantArrayType( | |||
3431 | getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal, | |||
3432 | /*IndexTypeQuals=*/0); | |||
3433 | BPVD = createImplicitFirstprivateForType( | |||
3434 | getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc()); | |||
3435 | PVD = createImplicitFirstprivateForType( | |||
3436 | getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc()); | |||
3437 | QualType SizesType = getContext().getConstantArrayType( | |||
3438 | getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1), | |||
3439 | ArrSize, nullptr, ArrayType::Normal, | |||
3440 | /*IndexTypeQuals=*/0); | |||
3441 | SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD, | |||
3442 | S.getBeginLoc()); | |||
3443 | TargetScope.addPrivate( | |||
3444 | BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; }); | |||
3445 | TargetScope.addPrivate(PVD, | |||
3446 | [&InputInfo]() { return InputInfo.PointersArray; }); | |||
3447 | TargetScope.addPrivate(SVD, | |||
3448 | [&InputInfo]() { return InputInfo.SizesArray; }); | |||
3449 | } | |||
3450 | (void)TargetScope.Privatize(); | |||
3451 | // Build list of dependences. | |||
3452 | for (const auto *C : S.getClausesOfKind<OMPDependClause>()) | |||
3453 | for (const Expr *IRef : C->varlists()) | |||
3454 | Data.Dependences.emplace_back(C->getDependencyKind(), IRef); | |||
3455 | auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, | |||
3456 | &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3457 | // Set proper addresses for generated private copies. | |||
3458 | OMPPrivateScope Scope(CGF); | |||
3459 | if (!Data.FirstprivateVars.empty()) { | |||
3460 | llvm::FunctionType *CopyFnTy = llvm::FunctionType::get( | |||
3461 | CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); | |||
3462 | enum { PrivatesParam = 2, CopyFnParam = 3 }; | |||
3463 | llvm::Value *CopyFn = CGF.Builder.CreateLoad( | |||
3464 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); | |||
3465 | llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( | |||
3466 | CS->getCapturedDecl()->getParam(PrivatesParam))); | |||
3467 | // Map privates. | |||
3468 | llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs; | |||
3469 | llvm::SmallVector<llvm::Value *, 16> CallArgs; | |||
3470 | CallArgs.push_back(PrivatesPtr); | |||
3471 | for (const Expr *E : Data.FirstprivateVars) { | |||
3472 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
3473 | Address PrivatePtr = | |||
3474 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), | |||
3475 | ".firstpriv.ptr.addr"); | |||
3476 | PrivatePtrs.emplace_back(VD, PrivatePtr); | |||
3477 | CallArgs.push_back(PrivatePtr.getPointer()); | |||
3478 | } | |||
3479 | CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( | |||
3480 | CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); | |||
3481 | for (const auto &Pair : PrivatePtrs) { | |||
3482 | Address Replacement(CGF.Builder.CreateLoad(Pair.second), | |||
3483 | CGF.getContext().getDeclAlign(Pair.first)); | |||
3484 | Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); | |||
3485 | } | |||
3486 | } | |||
3487 | // Privatize all private variables except for in_reduction items. | |||
3488 | (void)Scope.Privatize(); | |||
3489 | if (InputInfo.NumberOfTargetItems > 0) { | |||
3490 | InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( | |||
3491 | CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); | |||
3492 | InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( | |||
3493 | CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); | |||
3494 | InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( | |||
3495 | CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); | |||
3496 | } | |||
3497 | ||||
3498 | Action.Enter(CGF); | |||
3499 | OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); | |||
3500 | BodyGen(CGF); | |||
3501 | }; | |||
3502 | llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( | |||
3503 | S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true, | |||
3504 | Data.NumberOfParts); | |||
3505 | llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0); | |||
3506 | IntegerLiteral IfCond(getContext(), TrueOrFalse, | |||
3507 | getContext().getIntTypeForBitwidth(32, /*Signed=*/0), | |||
3508 | SourceLocation()); | |||
3509 | ||||
3510 | CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn, | |||
3511 | SharedsTy, CapturedStruct, &IfCond, Data); | |||
3512 | } | |||
3513 | ||||
3514 | void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { | |||
3515 | // Emit outlined function for task construct. | |||
3516 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); | |||
3517 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); | |||
3518 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); | |||
3519 | const Expr *IfCond = nullptr; | |||
3520 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
3521 | if (C->getNameModifier() == OMPD_unknown || | |||
3522 | C->getNameModifier() == OMPD_task) { | |||
3523 | IfCond = C->getCondition(); | |||
3524 | break; | |||
3525 | } | |||
3526 | } | |||
3527 | ||||
3528 | OMPTaskDataTy Data; | |||
3529 | // Check if we should emit tied or untied task. | |||
3530 | Data.Tied = !S.getSingleClause<OMPUntiedClause>(); | |||
3531 | auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { | |||
3532 | CGF.EmitStmt(CS->getCapturedStmt()); | |||
3533 | }; | |||
3534 | auto &&TaskGen = [&S, SharedsTy, CapturedStruct, | |||
3535 | IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, | |||
3536 | const OMPTaskDataTy &Data) { | |||
3537 | CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, | |||
3538 | SharedsTy, CapturedStruct, IfCond, | |||
3539 | Data); | |||
3540 | }; | |||
3541 | EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data); | |||
3542 | } | |||
3543 | ||||
3544 | void CodeGenFunction::EmitOMPTaskyieldDirective( | |||
3545 | const OMPTaskyieldDirective &S) { | |||
3546 | CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc()); | |||
3547 | } | |||
3548 | ||||
3549 | void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { | |||
3550 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier); | |||
3551 | } | |||
3552 | ||||
3553 | void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { | |||
3554 | CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc()); | |||
3555 | } | |||
3556 | ||||
3557 | void CodeGenFunction::EmitOMPTaskgroupDirective( | |||
3558 | const OMPTaskgroupDirective &S) { | |||
3559 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
3560 | Action.Enter(CGF); | |||
3561 | if (const Expr *E = S.getReductionRef()) { | |||
3562 | SmallVector<const Expr *, 4> LHSs; | |||
3563 | SmallVector<const Expr *, 4> RHSs; | |||
3564 | OMPTaskDataTy Data; | |||
3565 | for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) { | |||
3566 | auto IPriv = C->privates().begin(); | |||
3567 | auto IRed = C->reduction_ops().begin(); | |||
3568 | auto ILHS = C->lhs_exprs().begin(); | |||
3569 | auto IRHS = C->rhs_exprs().begin(); | |||
3570 | for (const Expr *Ref : C->varlists()) { | |||
3571 | Data.ReductionVars.emplace_back(Ref); | |||
3572 | Data.ReductionCopies.emplace_back(*IPriv); | |||
3573 | Data.ReductionOps.emplace_back(*IRed); | |||
3574 | LHSs.emplace_back(*ILHS); | |||
3575 | RHSs.emplace_back(*IRHS); | |||
3576 | std::advance(IPriv, 1); | |||
3577 | std::advance(IRed, 1); | |||
3578 | std::advance(ILHS, 1); | |||
3579 | std::advance(IRHS, 1); | |||
3580 | } | |||
3581 | } | |||
3582 | llvm::Value *ReductionDesc = | |||
3583 | CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), | |||
3584 | LHSs, RHSs, Data); | |||
3585 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
3586 | CGF.EmitVarDecl(*VD); | |||
3587 | CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), | |||
3588 | /*Volatile=*/false, E->getType()); | |||
3589 | } | |||
3590 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
3591 | }; | |||
3592 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3593 | CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc()); | |||
3594 | } | |||
3595 | ||||
3596 | void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { | |||
3597 | CGM.getOpenMPRuntime().emitFlush( | |||
3598 | *this, | |||
3599 | [&S]() -> ArrayRef<const Expr *> { | |||
3600 | if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) | |||
3601 | return llvm::makeArrayRef(FlushClause->varlist_begin(), | |||
3602 | FlushClause->varlist_end()); | |||
3603 | return llvm::None; | |||
3604 | }(), | |||
3605 | S.getBeginLoc()); | |||
3606 | } | |||
3607 | ||||
3608 | void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S, | |||
3609 | const CodeGenLoopTy &CodeGenLoop, | |||
3610 | Expr *IncExpr) { | |||
3611 | // Emit the loop iteration variable. | |||
3612 | const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); | |||
3613 | const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl()); | |||
3614 | EmitVarDecl(*IVDecl); | |||
3615 | ||||
3616 | // Emit the iterations count variable. | |||
3617 | // If it is not a variable, Sema decided to calculate iterations count on each | |||
3618 | // iteration (e.g., it is foldable into a constant). | |||
3619 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { | |||
3620 | EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); | |||
3621 | // Emit calculation of the iterations count. | |||
3622 | EmitIgnoredExpr(S.getCalcLastIteration()); | |||
3623 | } | |||
3624 | ||||
3625 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); | |||
3626 | ||||
3627 | bool HasLastprivateClause = false; | |||
3628 | // Check pre-condition. | |||
3629 | { | |||
3630 | OMPLoopScope PreInitScope(*this, S); | |||
3631 | // Skip the entire loop if we don't meet the precondition. | |||
3632 | // If the condition constant folds and can be elided, avoid emitting the | |||
3633 | // whole loop. | |||
3634 | bool CondConstant; | |||
3635 | llvm::BasicBlock *ContBlock = nullptr; | |||
3636 | if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { | |||
3637 | if (!CondConstant) | |||
3638 | return; | |||
3639 | } else { | |||
3640 | llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); | |||
3641 | ContBlock = createBasicBlock("omp.precond.end"); | |||
3642 | emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, | |||
3643 | getProfileCount(&S)); | |||
3644 | EmitBlock(ThenBlock); | |||
3645 | incrementProfileCounter(&S); | |||
3646 | } | |||
3647 | ||||
3648 | emitAlignedClause(*this, S); | |||
3649 | // Emit 'then' code. | |||
3650 | { | |||
3651 | // Emit helper vars inits. | |||
3652 | ||||
3653 | LValue LB = EmitOMPHelperVar( | |||
3654 | *this, cast<DeclRefExpr>( | |||
3655 | (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
3656 | ? S.getCombinedLowerBoundVariable() | |||
3657 | : S.getLowerBoundVariable()))); | |||
3658 | LValue UB = EmitOMPHelperVar( | |||
3659 | *this, cast<DeclRefExpr>( | |||
3660 | (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
3661 | ? S.getCombinedUpperBoundVariable() | |||
3662 | : S.getUpperBoundVariable()))); | |||
3663 | LValue ST = | |||
3664 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable())); | |||
3665 | LValue IL = | |||
3666 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); | |||
3667 | ||||
3668 | OMPPrivateScope LoopScope(*this); | |||
3669 | if (EmitOMPFirstprivateClause(S, LoopScope)) { | |||
3670 | // Emit implicit barrier to synchronize threads and avoid data races | |||
3671 | // on initialization of firstprivate variables and post-update of | |||
3672 | // lastprivate variables. | |||
3673 | CGM.getOpenMPRuntime().emitBarrierCall( | |||
3674 | *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, | |||
3675 | /*ForceSimpleCall=*/true); | |||
3676 | } | |||
3677 | EmitOMPPrivateClause(S, LoopScope); | |||
3678 | if (isOpenMPSimdDirective(S.getDirectiveKind()) && | |||
3679 | !isOpenMPParallelDirective(S.getDirectiveKind()) && | |||
3680 | !isOpenMPTeamsDirective(S.getDirectiveKind())) | |||
3681 | EmitOMPReductionClauseInit(S, LoopScope); | |||
3682 | HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); | |||
3683 | EmitOMPPrivateLoopCounters(S, LoopScope); | |||
3684 | (void)LoopScope.Privatize(); | |||
3685 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
3686 | CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); | |||
3687 | ||||
3688 | // Detect the distribute schedule kind and chunk. | |||
3689 | llvm::Value *Chunk = nullptr; | |||
3690 | OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown; | |||
3691 | if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) { | |||
3692 | ScheduleKind = C->getDistScheduleKind(); | |||
3693 | if (const Expr *Ch = C->getChunkSize()) { | |||
3694 | Chunk = EmitScalarExpr(Ch); | |||
3695 | Chunk = EmitScalarConversion(Chunk, Ch->getType(), | |||
3696 | S.getIterationVariable()->getType(), | |||
3697 | S.getBeginLoc()); | |||
3698 | } | |||
3699 | } else { | |||
3700 | // Default behaviour for dist_schedule clause. | |||
3701 | CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk( | |||
3702 | *this, S, ScheduleKind, Chunk); | |||
3703 | } | |||
3704 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); | |||
3705 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); | |||
3706 | ||||
3707 | // OpenMP [2.10.8, distribute Construct, Description] | |||
3708 | // If dist_schedule is specified, kind must be static. If specified, | |||
3709 | // iterations are divided into chunks of size chunk_size, chunks are | |||
3710 | // assigned to the teams of the league in a round-robin fashion in the | |||
3711 | // order of the team number. When no chunk_size is specified, the | |||
3712 | // iteration space is divided into chunks that are approximately equal | |||
3713 | // in size, and at most one chunk is distributed to each team of the | |||
3714 | // league. The size of the chunks is unspecified in this case. | |||
3715 | bool StaticChunked = RT.isStaticChunked( | |||
3716 | ScheduleKind, /* Chunked */ Chunk != nullptr) && | |||
3717 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()); | |||
3718 | if (RT.isStaticNonchunked(ScheduleKind, | |||
3719 | /* Chunked */ Chunk != nullptr) || | |||
3720 | StaticChunked) { | |||
3721 | CGOpenMPRuntime::StaticRTInput StaticInit( | |||
3722 | IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this), | |||
3723 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), | |||
3724 | StaticChunked ? Chunk : nullptr); | |||
3725 | RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, | |||
3726 | StaticInit); | |||
3727 | JumpDest LoopExit = | |||
3728 | getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); | |||
3729 | // UB = min(UB, GlobalUB); | |||
3730 | EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
3731 | ? S.getCombinedEnsureUpperBound() | |||
3732 | : S.getEnsureUpperBound()); | |||
3733 | // IV = LB; | |||
3734 | EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
3735 | ? S.getCombinedInit() | |||
3736 | : S.getInit()); | |||
3737 | ||||
3738 | const Expr *Cond = | |||
3739 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) | |||
3740 | ? S.getCombinedCond() | |||
3741 | : S.getCond(); | |||
3742 | ||||
3743 | if (StaticChunked) | |||
3744 | Cond = S.getCombinedDistCond(); | |||
3745 | ||||
3746 | // For static unchunked schedules generate: | |||
3747 | // | |||
3748 | // 1. For distribute alone, codegen | |||
3749 | // while (idx <= UB) { | |||
3750 | // BODY; | |||
3751 | // ++idx; | |||
3752 | // } | |||
3753 | // | |||
3754 | // 2. When combined with 'for' (e.g. as in 'distribute parallel for') | |||
3755 | // while (idx <= UB) { | |||
3756 | // <CodeGen rest of pragma>(LB, UB); | |||
3757 | // idx += ST; | |||
3758 | // } | |||
3759 | // | |||
3760 | // For static chunk one schedule generate: | |||
3761 | // | |||
3762 | // while (IV <= GlobalUB) { | |||
3763 | // <CodeGen rest of pragma>(LB, UB); | |||
3764 | // LB += ST; | |||
3765 | // UB += ST; | |||
3766 | // UB = min(UB, GlobalUB); | |||
3767 | // IV = LB; | |||
3768 | // } | |||
3769 | // | |||
3770 | emitCommonSimdLoop( | |||
3771 | *this, S, | |||
3772 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
3773 | if (isOpenMPSimdDirective(S.getDirectiveKind())) | |||
3774 | CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); | |||
3775 | }, | |||
3776 | [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop, | |||
3777 | StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { | |||
3778 | CGF.EmitOMPInnerLoop( | |||
3779 | S, LoopScope.requiresCleanups(), Cond, IncExpr, | |||
3780 | [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { | |||
3781 | CodeGenLoop(CGF, S, LoopExit); | |||
3782 | }, | |||
3783 | [&S, StaticChunked](CodeGenFunction &CGF) { | |||
3784 | if (StaticChunked) { | |||
3785 | CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); | |||
3786 | CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); | |||
3787 | CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); | |||
3788 | CGF.EmitIgnoredExpr(S.getCombinedInit()); | |||
3789 | } | |||
3790 | }); | |||
3791 | }); | |||
3792 | EmitBlock(LoopExit.getBlock()); | |||
3793 | // Tell the runtime we are done. | |||
3794 | RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind()); | |||
3795 | } else { | |||
3796 | // Emit the outer loop, which requests its work chunk [LB..UB] from | |||
3797 | // runtime and runs the inner loop to process it. | |||
3798 | const OMPLoopArguments LoopArguments = { | |||
3799 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), | |||
3800 | IL.getAddress(*this), Chunk}; | |||
3801 | EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments, | |||
3802 | CodeGenLoop); | |||
3803 | } | |||
3804 | if (isOpenMPSimdDirective(S.getDirectiveKind())) { | |||
3805 | EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { | |||
3806 | return CGF.Builder.CreateIsNotNull( | |||
3807 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
3808 | }); | |||
3809 | } | |||
3810 | if (isOpenMPSimdDirective(S.getDirectiveKind()) && | |||
3811 | !isOpenMPParallelDirective(S.getDirectiveKind()) && | |||
3812 | !isOpenMPTeamsDirective(S.getDirectiveKind())) { | |||
3813 | EmitOMPReductionClauseFinal(S, OMPD_simd); | |||
3814 | // Emit post-update of the reduction variables if IsLastIter != 0. | |||
3815 | emitPostUpdateForReductionClause( | |||
3816 | *this, S, [IL, &S](CodeGenFunction &CGF) { | |||
3817 | return CGF.Builder.CreateIsNotNull( | |||
3818 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); | |||
3819 | }); | |||
3820 | } | |||
3821 | // Emit final copy of the lastprivate variables if IsLastIter != 0. | |||
3822 | if (HasLastprivateClause) { | |||
3823 | EmitOMPLastprivateClauseFinal( | |||
3824 | S, /*NoFinals=*/false, | |||
3825 | Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); | |||
3826 | } | |||
3827 | } | |||
3828 | ||||
3829 | // We're now done with the loop, so jump to the continuation block. | |||
3830 | if (ContBlock) { | |||
3831 | EmitBranch(ContBlock); | |||
3832 | EmitBlock(ContBlock, true); | |||
3833 | } | |||
3834 | } | |||
3835 | } | |||
3836 | ||||
3837 | void CodeGenFunction::EmitOMPDistributeDirective( | |||
3838 | const OMPDistributeDirective &S) { | |||
3839 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
3840 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
3841 | }; | |||
3842 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3843 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); | |||
3844 | } | |||
3845 | ||||
3846 | static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM, | |||
3847 | const CapturedStmt *S) { | |||
3848 | CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); | |||
3849 | CodeGenFunction::CGCapturedStmtInfo CapStmtInfo; | |||
3850 | CGF.CapturedStmtInfo = &CapStmtInfo; | |||
3851 | llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); | |||
3852 | Fn->setDoesNotRecurse(); | |||
3853 | return Fn; | |||
3854 | } | |||
3855 | ||||
3856 | void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { | |||
3857 | if (S.hasClausesOfKind<OMPDependClause>()) { | |||
3858 | assert(!S.getAssociatedStmt() &&((!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct." ) ? static_cast<void> (0) : __assert_fail ("!S.getAssociatedStmt() && \"No associated statement must be in ordered depend construct.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3859, __PRETTY_FUNCTION__)) | |||
3859 | "No associated statement must be in ordered depend construct.")((!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct." ) ? static_cast<void> (0) : __assert_fail ("!S.getAssociatedStmt() && \"No associated statement must be in ordered depend construct.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3859, __PRETTY_FUNCTION__)); | |||
3860 | for (const auto *DC : S.getClausesOfKind<OMPDependClause>()) | |||
3861 | CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); | |||
3862 | return; | |||
3863 | } | |||
3864 | const auto *C = S.getSingleClause<OMPSIMDClause>(); | |||
3865 | auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, | |||
3866 | PrePostActionTy &Action) { | |||
3867 | const CapturedStmt *CS = S.getInnermostCapturedStmt(); | |||
3868 | if (C) { | |||
3869 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; | |||
3870 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); | |||
3871 | llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); | |||
3872 | CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), | |||
3873 | OutlinedFn, CapturedVars); | |||
3874 | } else { | |||
3875 | Action.Enter(CGF); | |||
3876 | CGF.EmitStmt(CS->getCapturedStmt()); | |||
3877 | } | |||
3878 | }; | |||
3879 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
3880 | CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C); | |||
3881 | } | |||
3882 | ||||
3883 | static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, | |||
3884 | QualType SrcType, QualType DestType, | |||
3885 | SourceLocation Loc) { | |||
3886 | assert(CGF.hasScalarEvaluationKind(DestType) &&((CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind." ) ? static_cast<void> (0) : __assert_fail ("CGF.hasScalarEvaluationKind(DestType) && \"DestType must have scalar evaluation kind.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3887, __PRETTY_FUNCTION__)) | |||
3887 | "DestType must have scalar evaluation kind.")((CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind." ) ? static_cast<void> (0) : __assert_fail ("CGF.hasScalarEvaluationKind(DestType) && \"DestType must have scalar evaluation kind.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3887, __PRETTY_FUNCTION__)); | |||
3888 | assert(!Val.isAggregate() && "Must be a scalar or complex.")((!Val.isAggregate() && "Must be a scalar or complex." ) ? static_cast<void> (0) : __assert_fail ("!Val.isAggregate() && \"Must be a scalar or complex.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3888, __PRETTY_FUNCTION__)); | |||
3889 | return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, | |||
3890 | DestType, Loc) | |||
3891 | : CGF.EmitComplexToScalarConversion( | |||
3892 | Val.getComplexVal(), SrcType, DestType, Loc); | |||
3893 | } | |||
3894 | ||||
3895 | static CodeGenFunction::ComplexPairTy | |||
3896 | convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, | |||
3897 | QualType DestType, SourceLocation Loc) { | |||
3898 | assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&((CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind." ) ? static_cast<void> (0) : __assert_fail ("CGF.getEvaluationKind(DestType) == TEK_Complex && \"DestType must have complex evaluation kind.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3899, __PRETTY_FUNCTION__)) | |||
3899 | "DestType must have complex evaluation kind.")((CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind." ) ? static_cast<void> (0) : __assert_fail ("CGF.getEvaluationKind(DestType) == TEK_Complex && \"DestType must have complex evaluation kind.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3899, __PRETTY_FUNCTION__)); | |||
3900 | CodeGenFunction::ComplexPairTy ComplexVal; | |||
3901 | if (Val.isScalar()) { | |||
3902 | // Convert the input element to the element type of the complex. | |||
3903 | QualType DestElementType = | |||
3904 | DestType->castAs<ComplexType>()->getElementType(); | |||
3905 | llvm::Value *ScalarVal = CGF.EmitScalarConversion( | |||
3906 | Val.getScalarVal(), SrcType, DestElementType, Loc); | |||
3907 | ComplexVal = CodeGenFunction::ComplexPairTy( | |||
3908 | ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); | |||
3909 | } else { | |||
3910 | assert(Val.isComplex() && "Must be a scalar or complex.")((Val.isComplex() && "Must be a scalar or complex.") ? static_cast<void> (0) : __assert_fail ("Val.isComplex() && \"Must be a scalar or complex.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3910, __PRETTY_FUNCTION__)); | |||
3911 | QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); | |||
3912 | QualType DestElementType = | |||
3913 | DestType->castAs<ComplexType>()->getElementType(); | |||
3914 | ComplexVal.first = CGF.EmitScalarConversion( | |||
3915 | Val.getComplexVal().first, SrcElementType, DestElementType, Loc); | |||
3916 | ComplexVal.second = CGF.EmitScalarConversion( | |||
3917 | Val.getComplexVal().second, SrcElementType, DestElementType, Loc); | |||
3918 | } | |||
3919 | return ComplexVal; | |||
3920 | } | |||
3921 | ||||
3922 | static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, | |||
3923 | LValue LVal, RValue RVal) { | |||
3924 | if (LVal.isGlobalReg()) { | |||
3925 | CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); | |||
3926 | } else { | |||
3927 | CGF.EmitAtomicStore(RVal, LVal, | |||
3928 | IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent | |||
3929 | : llvm::AtomicOrdering::Monotonic, | |||
3930 | LVal.isVolatile(), /*isInit=*/false); | |||
3931 | } | |||
3932 | } | |||
3933 | ||||
3934 | void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal, | |||
3935 | QualType RValTy, SourceLocation Loc) { | |||
3936 | switch (getEvaluationKind(LVal.getType())) { | |||
3937 | case TEK_Scalar: | |||
3938 | EmitStoreThroughLValue(RValue::get(convertToScalarValue( | |||
3939 | *this, RVal, RValTy, LVal.getType(), Loc)), | |||
3940 | LVal); | |||
3941 | break; | |||
3942 | case TEK_Complex: | |||
3943 | EmitStoreOfComplex( | |||
3944 | convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal, | |||
3945 | /*isInit=*/false); | |||
3946 | break; | |||
3947 | case TEK_Aggregate: | |||
3948 | llvm_unreachable("Must be a scalar or complex.")::llvm::llvm_unreachable_internal("Must be a scalar or complex." , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3948); | |||
3949 | } | |||
3950 | } | |||
3951 | ||||
3952 | static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, | |||
3953 | const Expr *X, const Expr *V, | |||
3954 | SourceLocation Loc) { | |||
3955 | // v = x; | |||
3956 | assert(V->isLValue() && "V of 'omp atomic read' is not lvalue")((V->isLValue() && "V of 'omp atomic read' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("V->isLValue() && \"V of 'omp atomic read' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3956, __PRETTY_FUNCTION__)); | |||
3957 | assert(X->isLValue() && "X of 'omp atomic read' is not lvalue")((X->isLValue() && "X of 'omp atomic read' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic read' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3957, __PRETTY_FUNCTION__)); | |||
3958 | LValue XLValue = CGF.EmitLValue(X); | |||
3959 | LValue VLValue = CGF.EmitLValue(V); | |||
3960 | RValue Res = XLValue.isGlobalReg() | |||
3961 | ? CGF.EmitLoadOfLValue(XLValue, Loc) | |||
3962 | : CGF.EmitAtomicLoad( | |||
3963 | XLValue, Loc, | |||
3964 | IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent | |||
3965 | : llvm::AtomicOrdering::Monotonic, | |||
3966 | XLValue.isVolatile()); | |||
3967 | // OpenMP, 2.12.6, atomic Construct | |||
3968 | // Any atomic construct with a seq_cst clause forces the atomically | |||
3969 | // performed operation to include an implicit flush operation without a | |||
3970 | // list. | |||
3971 | if (IsSeqCst) | |||
3972 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); | |||
3973 | CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); | |||
3974 | } | |||
3975 | ||||
3976 | static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, | |||
3977 | const Expr *X, const Expr *E, | |||
3978 | SourceLocation Loc) { | |||
3979 | // x = expr; | |||
3980 | assert(X->isLValue() && "X of 'omp atomic write' is not lvalue")((X->isLValue() && "X of 'omp atomic write' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic write' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 3980, __PRETTY_FUNCTION__)); | |||
3981 | emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); | |||
3982 | // OpenMP, 2.12.6, atomic Construct | |||
3983 | // Any atomic construct with a seq_cst clause forces the atomically | |||
3984 | // performed operation to include an implicit flush operation without a | |||
3985 | // list. | |||
3986 | if (IsSeqCst) | |||
3987 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); | |||
3988 | } | |||
3989 | ||||
3990 | static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, | |||
3991 | RValue Update, | |||
3992 | BinaryOperatorKind BO, | |||
3993 | llvm::AtomicOrdering AO, | |||
3994 | bool IsXLHSInRHSPart) { | |||
3995 | ASTContext &Context = CGF.getContext(); | |||
3996 | // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' | |||
3997 | // expression is simple and atomic is allowed for the given type for the | |||
3998 | // target platform. | |||
3999 | if (BO == BO_Comma || !Update.isScalar() || | |||
4000 | !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() || | |||
4001 | (!isa<llvm::ConstantInt>(Update.getScalarVal()) && | |||
4002 | (Update.getScalarVal()->getType() != | |||
4003 | X.getAddress(CGF).getElementType())) || | |||
4004 | !X.getAddress(CGF).getElementType()->isIntegerTy() || | |||
4005 | !Context.getTargetInfo().hasBuiltinAtomic( | |||
4006 | Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) | |||
4007 | return std::make_pair(false, RValue::get(nullptr)); | |||
4008 | ||||
4009 | llvm::AtomicRMWInst::BinOp RMWOp; | |||
4010 | switch (BO) { | |||
4011 | case BO_Add: | |||
4012 | RMWOp = llvm::AtomicRMWInst::Add; | |||
4013 | break; | |||
4014 | case BO_Sub: | |||
4015 | if (!IsXLHSInRHSPart) | |||
4016 | return std::make_pair(false, RValue::get(nullptr)); | |||
4017 | RMWOp = llvm::AtomicRMWInst::Sub; | |||
4018 | break; | |||
4019 | case BO_And: | |||
4020 | RMWOp = llvm::AtomicRMWInst::And; | |||
4021 | break; | |||
4022 | case BO_Or: | |||
4023 | RMWOp = llvm::AtomicRMWInst::Or; | |||
4024 | break; | |||
4025 | case BO_Xor: | |||
4026 | RMWOp = llvm::AtomicRMWInst::Xor; | |||
4027 | break; | |||
4028 | case BO_LT: | |||
4029 | RMWOp = X.getType()->hasSignedIntegerRepresentation() | |||
4030 | ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min | |||
4031 | : llvm::AtomicRMWInst::Max) | |||
4032 | : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin | |||
4033 | : llvm::AtomicRMWInst::UMax); | |||
4034 | break; | |||
4035 | case BO_GT: | |||
4036 | RMWOp = X.getType()->hasSignedIntegerRepresentation() | |||
4037 | ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max | |||
4038 | : llvm::AtomicRMWInst::Min) | |||
4039 | : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax | |||
4040 | : llvm::AtomicRMWInst::UMin); | |||
4041 | break; | |||
4042 | case BO_Assign: | |||
4043 | RMWOp = llvm::AtomicRMWInst::Xchg; | |||
4044 | break; | |||
4045 | case BO_Mul: | |||
4046 | case BO_Div: | |||
4047 | case BO_Rem: | |||
4048 | case BO_Shl: | |||
4049 | case BO_Shr: | |||
4050 | case BO_LAnd: | |||
4051 | case BO_LOr: | |||
4052 | return std::make_pair(false, RValue::get(nullptr)); | |||
4053 | case BO_PtrMemD: | |||
4054 | case BO_PtrMemI: | |||
4055 | case BO_LE: | |||
4056 | case BO_GE: | |||
4057 | case BO_EQ: | |||
4058 | case BO_NE: | |||
4059 | case BO_Cmp: | |||
4060 | case BO_AddAssign: | |||
4061 | case BO_SubAssign: | |||
4062 | case BO_AndAssign: | |||
4063 | case BO_OrAssign: | |||
4064 | case BO_XorAssign: | |||
4065 | case BO_MulAssign: | |||
4066 | case BO_DivAssign: | |||
4067 | case BO_RemAssign: | |||
4068 | case BO_ShlAssign: | |||
4069 | case BO_ShrAssign: | |||
4070 | case BO_Comma: | |||
4071 | llvm_unreachable("Unsupported atomic update operation")::llvm::llvm_unreachable_internal("Unsupported atomic update operation" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4071); | |||
4072 | } | |||
4073 | llvm::Value *UpdateVal = Update.getScalarVal(); | |||
4074 | if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { | |||
4075 | UpdateVal = CGF.Builder.CreateIntCast( | |||
4076 | IC, X.getAddress(CGF).getElementType(), | |||
4077 | X.getType()->hasSignedIntegerRepresentation()); | |||
4078 | } | |||
4079 | llvm::Value *Res = | |||
4080 | CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); | |||
4081 | return std::make_pair(true, RValue::get(Res)); | |||
4082 | } | |||
4083 | ||||
4084 | std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( | |||
4085 | LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, | |||
4086 | llvm::AtomicOrdering AO, SourceLocation Loc, | |||
4087 | const llvm::function_ref<RValue(RValue)> CommonGen) { | |||
4088 | // Update expressions are allowed to have the following forms: | |||
4089 | // x binop= expr; -> xrval + expr; | |||
4090 | // x++, ++x -> xrval + 1; | |||
4091 | // x--, --x -> xrval - 1; | |||
4092 | // x = x binop expr; -> xrval binop expr | |||
4093 | // x = expr Op x; - > expr binop xrval; | |||
4094 | auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart); | |||
4095 | if (!Res.first) { | |||
4096 | if (X.isGlobalReg()) { | |||
4097 | // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop | |||
4098 | // 'xrval'. | |||
4099 | EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); | |||
4100 | } else { | |||
4101 | // Perform compare-and-swap procedure. | |||
4102 | EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); | |||
4103 | } | |||
4104 | } | |||
4105 | return Res; | |||
4106 | } | |||
4107 | ||||
4108 | static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, | |||
4109 | const Expr *X, const Expr *E, | |||
4110 | const Expr *UE, bool IsXLHSInRHSPart, | |||
4111 | SourceLocation Loc) { | |||
4112 | assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&((isa<BinaryOperator>(UE->IgnoreImpCasts()) && "Update expr in 'atomic update' must be a binary operator.") ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic update' must be a binary operator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4113, __PRETTY_FUNCTION__)) | |||
4113 | "Update expr in 'atomic update' must be a binary operator.")((isa<BinaryOperator>(UE->IgnoreImpCasts()) && "Update expr in 'atomic update' must be a binary operator.") ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic update' must be a binary operator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4113, __PRETTY_FUNCTION__)); | |||
4114 | const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); | |||
4115 | // Update expressions are allowed to have the following forms: | |||
4116 | // x binop= expr; -> xrval + expr; | |||
4117 | // x++, ++x -> xrval + 1; | |||
4118 | // x--, --x -> xrval - 1; | |||
4119 | // x = x binop expr; -> xrval binop expr | |||
4120 | // x = expr Op x; - > expr binop xrval; | |||
4121 | assert(X->isLValue() && "X of 'omp atomic update' is not lvalue")((X->isLValue() && "X of 'omp atomic update' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic update' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4121, __PRETTY_FUNCTION__)); | |||
4122 | LValue XLValue = CGF.EmitLValue(X); | |||
4123 | RValue ExprRValue = CGF.EmitAnyExpr(E); | |||
4124 | llvm::AtomicOrdering AO = IsSeqCst | |||
4125 | ? llvm::AtomicOrdering::SequentiallyConsistent | |||
4126 | : llvm::AtomicOrdering::Monotonic; | |||
4127 | const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); | |||
4128 | const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); | |||
4129 | const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; | |||
4130 | const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; | |||
4131 | auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { | |||
4132 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); | |||
4133 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); | |||
4134 | return CGF.EmitAnyExpr(UE); | |||
4135 | }; | |||
4136 | (void)CGF.EmitOMPAtomicSimpleUpdateExpr( | |||
4137 | XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); | |||
4138 | // OpenMP, 2.12.6, atomic Construct | |||
4139 | // Any atomic construct with a seq_cst clause forces the atomically | |||
4140 | // performed operation to include an implicit flush operation without a | |||
4141 | // list. | |||
4142 | if (IsSeqCst) | |||
4143 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); | |||
4144 | } | |||
4145 | ||||
4146 | static RValue convertToType(CodeGenFunction &CGF, RValue Value, | |||
4147 | QualType SourceType, QualType ResType, | |||
4148 | SourceLocation Loc) { | |||
4149 | switch (CGF.getEvaluationKind(ResType)) { | |||
4150 | case TEK_Scalar: | |||
4151 | return RValue::get( | |||
4152 | convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); | |||
4153 | case TEK_Complex: { | |||
4154 | auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); | |||
4155 | return RValue::getComplex(Res.first, Res.second); | |||
4156 | } | |||
4157 | case TEK_Aggregate: | |||
4158 | break; | |||
4159 | } | |||
4160 | llvm_unreachable("Must be a scalar or complex.")::llvm::llvm_unreachable_internal("Must be a scalar or complex." , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4160); | |||
4161 | } | |||
4162 | ||||
4163 | static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, | |||
4164 | bool IsPostfixUpdate, const Expr *V, | |||
4165 | const Expr *X, const Expr *E, | |||
4166 | const Expr *UE, bool IsXLHSInRHSPart, | |||
4167 | SourceLocation Loc) { | |||
4168 | assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue")((X->isLValue() && "X of 'omp atomic capture' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("X->isLValue() && \"X of 'omp atomic capture' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4168, __PRETTY_FUNCTION__)); | |||
4169 | assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue")((V->isLValue() && "V of 'omp atomic capture' is not lvalue" ) ? static_cast<void> (0) : __assert_fail ("V->isLValue() && \"V of 'omp atomic capture' is not lvalue\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4169, __PRETTY_FUNCTION__)); | |||
4170 | RValue NewVVal; | |||
4171 | LValue VLValue = CGF.EmitLValue(V); | |||
4172 | LValue XLValue = CGF.EmitLValue(X); | |||
4173 | RValue ExprRValue = CGF.EmitAnyExpr(E); | |||
4174 | llvm::AtomicOrdering AO = IsSeqCst | |||
4175 | ? llvm::AtomicOrdering::SequentiallyConsistent | |||
4176 | : llvm::AtomicOrdering::Monotonic; | |||
4177 | QualType NewVValType; | |||
4178 | if (UE) { | |||
4179 | // 'x' is updated with some additional value. | |||
4180 | assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&((isa<BinaryOperator>(UE->IgnoreImpCasts()) && "Update expr in 'atomic capture' must be a binary operator." ) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic capture' must be a binary operator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4181, __PRETTY_FUNCTION__)) | |||
4181 | "Update expr in 'atomic capture' must be a binary operator.")((isa<BinaryOperator>(UE->IgnoreImpCasts()) && "Update expr in 'atomic capture' must be a binary operator." ) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(UE->IgnoreImpCasts()) && \"Update expr in 'atomic capture' must be a binary operator.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4181, __PRETTY_FUNCTION__)); | |||
4182 | const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); | |||
4183 | // Update expressions are allowed to have the following forms: | |||
4184 | // x binop= expr; -> xrval + expr; | |||
4185 | // x++, ++x -> xrval + 1; | |||
4186 | // x--, --x -> xrval - 1; | |||
4187 | // x = x binop expr; -> xrval binop expr | |||
4188 | // x = expr Op x; - > expr binop xrval; | |||
4189 | const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); | |||
4190 | const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); | |||
4191 | const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; | |||
4192 | NewVValType = XRValExpr->getType(); | |||
4193 | const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; | |||
4194 | auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, | |||
4195 | IsPostfixUpdate](RValue XRValue) { | |||
4196 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); | |||
4197 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); | |||
4198 | RValue Res = CGF.EmitAnyExpr(UE); | |||
4199 | NewVVal = IsPostfixUpdate ? XRValue : Res; | |||
4200 | return Res; | |||
4201 | }; | |||
4202 | auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( | |||
4203 | XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); | |||
4204 | if (Res.first) { | |||
4205 | // 'atomicrmw' instruction was generated. | |||
4206 | if (IsPostfixUpdate) { | |||
4207 | // Use old value from 'atomicrmw'. | |||
4208 | NewVVal = Res.second; | |||
4209 | } else { | |||
4210 | // 'atomicrmw' does not provide new value, so evaluate it using old | |||
4211 | // value of 'x'. | |||
4212 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); | |||
4213 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); | |||
4214 | NewVVal = CGF.EmitAnyExpr(UE); | |||
4215 | } | |||
4216 | } | |||
4217 | } else { | |||
4218 | // 'x' is simply rewritten with some 'expr'. | |||
4219 | NewVValType = X->getType().getNonReferenceType(); | |||
4220 | ExprRValue = convertToType(CGF, ExprRValue, E->getType(), | |||
4221 | X->getType().getNonReferenceType(), Loc); | |||
4222 | auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) { | |||
4223 | NewVVal = XRValue; | |||
4224 | return ExprRValue; | |||
4225 | }; | |||
4226 | // Try to perform atomicrmw xchg, otherwise simple exchange. | |||
4227 | auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( | |||
4228 | XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO, | |||
4229 | Loc, Gen); | |||
4230 | if (Res.first) { | |||
4231 | // 'atomicrmw' instruction was generated. | |||
4232 | NewVVal = IsPostfixUpdate ? Res.second : ExprRValue; | |||
4233 | } | |||
4234 | } | |||
4235 | // Emit post-update store to 'v' of old/new 'x' value. | |||
4236 | CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); | |||
4237 | // OpenMP, 2.12.6, atomic Construct | |||
4238 | // Any atomic construct with a seq_cst clause forces the atomically | |||
4239 | // performed operation to include an implicit flush operation without a | |||
4240 | // list. | |||
4241 | if (IsSeqCst) | |||
4242 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); | |||
4243 | } | |||
4244 | ||||
4245 | static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, | |||
4246 | bool IsSeqCst, bool IsPostfixUpdate, | |||
4247 | const Expr *X, const Expr *V, const Expr *E, | |||
4248 | const Expr *UE, bool IsXLHSInRHSPart, | |||
4249 | SourceLocation Loc) { | |||
4250 | switch (Kind) { | |||
4251 | case OMPC_read: | |||
4252 | emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); | |||
4253 | break; | |||
4254 | case OMPC_write: | |||
4255 | emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); | |||
4256 | break; | |||
4257 | case OMPC_unknown: | |||
4258 | case OMPC_update: | |||
4259 | emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); | |||
4260 | break; | |||
4261 | case OMPC_capture: | |||
4262 | emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, | |||
4263 | IsXLHSInRHSPart, Loc); | |||
4264 | break; | |||
4265 | case OMPC_if: | |||
4266 | case OMPC_final: | |||
4267 | case OMPC_num_threads: | |||
4268 | case OMPC_private: | |||
4269 | case OMPC_firstprivate: | |||
4270 | case OMPC_lastprivate: | |||
4271 | case OMPC_reduction: | |||
4272 | case OMPC_task_reduction: | |||
4273 | case OMPC_in_reduction: | |||
4274 | case OMPC_safelen: | |||
4275 | case OMPC_simdlen: | |||
4276 | case OMPC_allocator: | |||
4277 | case OMPC_allocate: | |||
4278 | case OMPC_collapse: | |||
4279 | case OMPC_default: | |||
4280 | case OMPC_seq_cst: | |||
4281 | case OMPC_shared: | |||
4282 | case OMPC_linear: | |||
4283 | case OMPC_aligned: | |||
4284 | case OMPC_copyin: | |||
4285 | case OMPC_copyprivate: | |||
4286 | case OMPC_flush: | |||
4287 | case OMPC_proc_bind: | |||
4288 | case OMPC_schedule: | |||
4289 | case OMPC_ordered: | |||
4290 | case OMPC_nowait: | |||
4291 | case OMPC_untied: | |||
4292 | case OMPC_threadprivate: | |||
4293 | case OMPC_depend: | |||
4294 | case OMPC_mergeable: | |||
4295 | case OMPC_device: | |||
4296 | case OMPC_threads: | |||
4297 | case OMPC_simd: | |||
4298 | case OMPC_map: | |||
4299 | case OMPC_num_teams: | |||
4300 | case OMPC_thread_limit: | |||
4301 | case OMPC_priority: | |||
4302 | case OMPC_grainsize: | |||
4303 | case OMPC_nogroup: | |||
4304 | case OMPC_num_tasks: | |||
4305 | case OMPC_hint: | |||
4306 | case OMPC_dist_schedule: | |||
4307 | case OMPC_defaultmap: | |||
4308 | case OMPC_uniform: | |||
4309 | case OMPC_to: | |||
4310 | case OMPC_from: | |||
4311 | case OMPC_use_device_ptr: | |||
4312 | case OMPC_is_device_ptr: | |||
4313 | case OMPC_unified_address: | |||
4314 | case OMPC_unified_shared_memory: | |||
4315 | case OMPC_reverse_offload: | |||
4316 | case OMPC_dynamic_allocators: | |||
4317 | case OMPC_atomic_default_mem_order: | |||
4318 | case OMPC_device_type: | |||
4319 | case OMPC_match: | |||
4320 | case OMPC_nontemporal: | |||
4321 | llvm_unreachable("Clause is not allowed in 'omp atomic'.")::llvm::llvm_unreachable_internal("Clause is not allowed in 'omp atomic'." , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4321); | |||
4322 | } | |||
4323 | } | |||
4324 | ||||
4325 | void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { | |||
4326 | bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>(); | |||
4327 | OpenMPClauseKind Kind = OMPC_unknown; | |||
4328 | for (const OMPClause *C : S.clauses()) { | |||
4329 | // Find first clause (skip seq_cst clause, if it is first). | |||
4330 | if (C->getClauseKind() != OMPC_seq_cst) { | |||
4331 | Kind = C->getClauseKind(); | |||
4332 | break; | |||
4333 | } | |||
4334 | } | |||
4335 | ||||
4336 | const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers(); | |||
4337 | if (const auto *FE = dyn_cast<FullExpr>(CS)) | |||
4338 | enterFullExpression(FE); | |||
4339 | // Processing for statements under 'atomic capture'. | |||
4340 | if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) { | |||
4341 | for (const Stmt *C : Compound->body()) { | |||
4342 | if (const auto *FE = dyn_cast<FullExpr>(C)) | |||
4343 | enterFullExpression(FE); | |||
4344 | } | |||
4345 | } | |||
4346 | ||||
4347 | auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF, | |||
4348 | PrePostActionTy &) { | |||
4349 | CGF.EmitStopPoint(CS); | |||
4350 | emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), | |||
4351 | S.getV(), S.getExpr(), S.getUpdateExpr(), | |||
4352 | S.isXLHSInRHSPart(), S.getBeginLoc()); | |||
4353 | }; | |||
4354 | OMPLexicalScope Scope(*this, S, OMPD_unknown); | |||
4355 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen); | |||
4356 | } | |||
4357 | ||||
4358 | static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, | |||
4359 | const OMPExecutableDirective &S, | |||
4360 | const RegionCodeGenTy &CodeGen) { | |||
4361 | assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()))((isOpenMPTargetExecutionDirective(S.getDirectiveKind())) ? static_cast <void> (0) : __assert_fail ("isOpenMPTargetExecutionDirective(S.getDirectiveKind())" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4361, __PRETTY_FUNCTION__)); | |||
4362 | CodeGenModule &CGM = CGF.CGM; | |||
4363 | ||||
4364 | // On device emit this construct as inlined code. | |||
4365 | if (CGM.getLangOpts().OpenMPIsDevice) { | |||
4366 | OMPLexicalScope Scope(CGF, S, OMPD_target); | |||
4367 | CGM.getOpenMPRuntime().emitInlinedDirective( | |||
4368 | CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4369 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
4370 | }); | |||
4371 | return; | |||
4372 | } | |||
4373 | ||||
4374 | llvm::Function *Fn = nullptr; | |||
4375 | llvm::Constant *FnID = nullptr; | |||
4376 | ||||
4377 | const Expr *IfCond = nullptr; | |||
4378 | // Check for the at most one if clause associated with the target region. | |||
4379 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
4380 | if (C->getNameModifier() == OMPD_unknown || | |||
4381 | C->getNameModifier() == OMPD_target) { | |||
4382 | IfCond = C->getCondition(); | |||
4383 | break; | |||
4384 | } | |||
4385 | } | |||
4386 | ||||
4387 | // Check if we have any device clause associated with the directive. | |||
4388 | const Expr *Device = nullptr; | |||
4389 | if (auto *C = S.getSingleClause<OMPDeviceClause>()) | |||
4390 | Device = C->getDevice(); | |||
4391 | ||||
4392 | // Check if we have an if clause whose conditional always evaluates to false | |||
4393 | // or if we do not have any targets specified. If so the target region is not | |||
4394 | // an offload entry point. | |||
4395 | bool IsOffloadEntry = true; | |||
4396 | if (IfCond) { | |||
4397 | bool Val; | |||
4398 | if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) | |||
4399 | IsOffloadEntry = false; | |||
4400 | } | |||
4401 | if (CGM.getLangOpts().OMPTargetTriples.empty()) | |||
4402 | IsOffloadEntry = false; | |||
4403 | ||||
4404 | assert(CGF.CurFuncDecl && "No parent declaration for target region!")((CGF.CurFuncDecl && "No parent declaration for target region!" ) ? static_cast<void> (0) : __assert_fail ("CGF.CurFuncDecl && \"No parent declaration for target region!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4404, __PRETTY_FUNCTION__)); | |||
4405 | StringRef ParentName; | |||
4406 | // In case we have Ctors/Dtors we use the complete type variant to produce | |||
4407 | // the mangling of the device outlined kernel. | |||
4408 | if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) | |||
4409 | ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete)); | |||
4410 | else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) | |||
4411 | ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete)); | |||
4412 | else | |||
4413 | ParentName = | |||
4414 | CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); | |||
4415 | ||||
4416 | // Emit target region as a standalone region. | |||
4417 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID, | |||
4418 | IsOffloadEntry, CodeGen); | |||
4419 | OMPLexicalScope Scope(CGF, S, OMPD_task); | |||
4420 | auto &&SizeEmitter = | |||
4421 | [IsOffloadEntry](CodeGenFunction &CGF, | |||
4422 | const OMPLoopDirective &D) -> llvm::Value * { | |||
4423 | if (IsOffloadEntry) { | |||
4424 | OMPLoopScope(CGF, D); | |||
4425 | // Emit calculation of the iterations count. | |||
4426 | llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); | |||
4427 | NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, | |||
4428 | /*isSigned=*/false); | |||
4429 | return NumIterations; | |||
4430 | } | |||
4431 | return nullptr; | |||
4432 | }; | |||
4433 | CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, | |||
4434 | SizeEmitter); | |||
4435 | } | |||
4436 | ||||
4437 | static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, | |||
4438 | PrePostActionTy &Action) { | |||
4439 | Action.Enter(CGF); | |||
4440 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4441 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
4442 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
4443 | (void)PrivateScope.Privatize(); | |||
4444 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
4445 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); | |||
4446 | ||||
4447 | CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); | |||
4448 | } | |||
4449 | ||||
4450 | void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM, | |||
4451 | StringRef ParentName, | |||
4452 | const OMPTargetDirective &S) { | |||
4453 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4454 | emitTargetRegion(CGF, S, Action); | |||
4455 | }; | |||
4456 | llvm::Function *Fn; | |||
4457 | llvm::Constant *Addr; | |||
4458 | // Emit target region as a standalone region. | |||
4459 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4460 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4461 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4461, __PRETTY_FUNCTION__)); | |||
4462 | } | |||
4463 | ||||
4464 | void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) { | |||
4465 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4466 | emitTargetRegion(CGF, S, Action); | |||
4467 | }; | |||
4468 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4469 | } | |||
4470 | ||||
4471 | static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, | |||
4472 | const OMPExecutableDirective &S, | |||
4473 | OpenMPDirectiveKind InnermostKind, | |||
4474 | const RegionCodeGenTy &CodeGen) { | |||
4475 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams); | |||
4476 | llvm::Function *OutlinedFn = | |||
4477 | CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( | |||
4478 | S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); | |||
4479 | ||||
4480 | const auto *NT = S.getSingleClause<OMPNumTeamsClause>(); | |||
4481 | const auto *TL = S.getSingleClause<OMPThreadLimitClause>(); | |||
4482 | if (NT || TL) { | |||
4483 | const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr; | |||
4484 | const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr; | |||
4485 | ||||
4486 | CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, | |||
4487 | S.getBeginLoc()); | |||
4488 | } | |||
4489 | ||||
4490 | OMPTeamsScope Scope(CGF, S); | |||
4491 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; | |||
4492 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); | |||
4493 | CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, | |||
4494 | CapturedVars); | |||
4495 | } | |||
4496 | ||||
4497 | void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) { | |||
4498 | // Emit teams region as a standalone region. | |||
4499 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4500 | Action.Enter(CGF); | |||
4501 | OMPPrivateScope PrivateScope(CGF); | |||
4502 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
4503 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
4504 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4505 | (void)PrivateScope.Privatize(); | |||
4506 | CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); | |||
4507 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4508 | }; | |||
4509 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); | |||
4510 | emitPostUpdateForReductionClause(*this, S, | |||
4511 | [](CodeGenFunction &) { return nullptr; }); | |||
4512 | } | |||
4513 | ||||
4514 | static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, | |||
4515 | const OMPTargetTeamsDirective &S) { | |||
4516 | auto *CS = S.getCapturedStmt(OMPD_teams); | |||
4517 | Action.Enter(CGF); | |||
4518 | // Emit teams region as a standalone region. | |||
4519 | auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4520 | Action.Enter(CGF); | |||
4521 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4522 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
4523 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
4524 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4525 | (void)PrivateScope.Privatize(); | |||
4526 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
4527 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); | |||
4528 | CGF.EmitStmt(CS->getCapturedStmt()); | |||
4529 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4530 | }; | |||
4531 | emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); | |||
4532 | emitPostUpdateForReductionClause(CGF, S, | |||
4533 | [](CodeGenFunction &) { return nullptr; }); | |||
4534 | } | |||
4535 | ||||
4536 | void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction( | |||
4537 | CodeGenModule &CGM, StringRef ParentName, | |||
4538 | const OMPTargetTeamsDirective &S) { | |||
4539 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4540 | emitTargetTeamsRegion(CGF, Action, S); | |||
4541 | }; | |||
4542 | llvm::Function *Fn; | |||
4543 | llvm::Constant *Addr; | |||
4544 | // Emit target region as a standalone region. | |||
4545 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4546 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4547 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4547, __PRETTY_FUNCTION__)); | |||
4548 | } | |||
4549 | ||||
4550 | void CodeGenFunction::EmitOMPTargetTeamsDirective( | |||
4551 | const OMPTargetTeamsDirective &S) { | |||
4552 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4553 | emitTargetTeamsRegion(CGF, Action, S); | |||
4554 | }; | |||
4555 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4556 | } | |||
4557 | ||||
4558 | static void | |||
4559 | emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, | |||
4560 | const OMPTargetTeamsDistributeDirective &S) { | |||
4561 | Action.Enter(CGF); | |||
4562 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4563 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
4564 | }; | |||
4565 | ||||
4566 | // Emit teams region as a standalone region. | |||
4567 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4568 | PrePostActionTy &Action) { | |||
4569 | Action.Enter(CGF); | |||
4570 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4571 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4572 | (void)PrivateScope.Privatize(); | |||
4573 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, | |||
4574 | CodeGenDistribute); | |||
4575 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4576 | }; | |||
4577 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); | |||
4578 | emitPostUpdateForReductionClause(CGF, S, | |||
4579 | [](CodeGenFunction &) { return nullptr; }); | |||
4580 | } | |||
4581 | ||||
4582 | void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( | |||
4583 | CodeGenModule &CGM, StringRef ParentName, | |||
4584 | const OMPTargetTeamsDistributeDirective &S) { | |||
4585 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4586 | emitTargetTeamsDistributeRegion(CGF, Action, S); | |||
4587 | }; | |||
4588 | llvm::Function *Fn; | |||
4589 | llvm::Constant *Addr; | |||
4590 | // Emit target region as a standalone region. | |||
4591 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4592 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4593 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4593, __PRETTY_FUNCTION__)); | |||
4594 | } | |||
4595 | ||||
4596 | void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( | |||
4597 | const OMPTargetTeamsDistributeDirective &S) { | |||
4598 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4599 | emitTargetTeamsDistributeRegion(CGF, Action, S); | |||
4600 | }; | |||
4601 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4602 | } | |||
4603 | ||||
4604 | static void emitTargetTeamsDistributeSimdRegion( | |||
4605 | CodeGenFunction &CGF, PrePostActionTy &Action, | |||
4606 | const OMPTargetTeamsDistributeSimdDirective &S) { | |||
4607 | Action.Enter(CGF); | |||
4608 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4609 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
4610 | }; | |||
4611 | ||||
4612 | // Emit teams region as a standalone region. | |||
4613 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4614 | PrePostActionTy &Action) { | |||
4615 | Action.Enter(CGF); | |||
4616 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4617 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4618 | (void)PrivateScope.Privatize(); | |||
4619 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, | |||
4620 | CodeGenDistribute); | |||
4621 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4622 | }; | |||
4623 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); | |||
4624 | emitPostUpdateForReductionClause(CGF, S, | |||
4625 | [](CodeGenFunction &) { return nullptr; }); | |||
4626 | } | |||
4627 | ||||
4628 | void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( | |||
4629 | CodeGenModule &CGM, StringRef ParentName, | |||
4630 | const OMPTargetTeamsDistributeSimdDirective &S) { | |||
4631 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4632 | emitTargetTeamsDistributeSimdRegion(CGF, Action, S); | |||
4633 | }; | |||
4634 | llvm::Function *Fn; | |||
4635 | llvm::Constant *Addr; | |||
4636 | // Emit target region as a standalone region. | |||
4637 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4638 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4639 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4639, __PRETTY_FUNCTION__)); | |||
4640 | } | |||
4641 | ||||
4642 | void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( | |||
4643 | const OMPTargetTeamsDistributeSimdDirective &S) { | |||
4644 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4645 | emitTargetTeamsDistributeSimdRegion(CGF, Action, S); | |||
4646 | }; | |||
4647 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4648 | } | |||
4649 | ||||
4650 | void CodeGenFunction::EmitOMPTeamsDistributeDirective( | |||
4651 | const OMPTeamsDistributeDirective &S) { | |||
4652 | ||||
4653 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4654 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
4655 | }; | |||
4656 | ||||
4657 | // Emit teams region as a standalone region. | |||
4658 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4659 | PrePostActionTy &Action) { | |||
4660 | Action.Enter(CGF); | |||
4661 | OMPPrivateScope PrivateScope(CGF); | |||
4662 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4663 | (void)PrivateScope.Privatize(); | |||
4664 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, | |||
4665 | CodeGenDistribute); | |||
4666 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4667 | }; | |||
4668 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); | |||
4669 | emitPostUpdateForReductionClause(*this, S, | |||
4670 | [](CodeGenFunction &) { return nullptr; }); | |||
4671 | } | |||
4672 | ||||
4673 | void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( | |||
4674 | const OMPTeamsDistributeSimdDirective &S) { | |||
4675 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4676 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); | |||
4677 | }; | |||
4678 | ||||
4679 | // Emit teams region as a standalone region. | |||
4680 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4681 | PrePostActionTy &Action) { | |||
4682 | Action.Enter(CGF); | |||
4683 | OMPPrivateScope PrivateScope(CGF); | |||
4684 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4685 | (void)PrivateScope.Privatize(); | |||
4686 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, | |||
4687 | CodeGenDistribute); | |||
4688 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4689 | }; | |||
4690 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen); | |||
4691 | emitPostUpdateForReductionClause(*this, S, | |||
4692 | [](CodeGenFunction &) { return nullptr; }); | |||
4693 | } | |||
4694 | ||||
4695 | void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( | |||
4696 | const OMPTeamsDistributeParallelForDirective &S) { | |||
4697 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4698 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
4699 | S.getDistInc()); | |||
4700 | }; | |||
4701 | ||||
4702 | // Emit teams region as a standalone region. | |||
4703 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4704 | PrePostActionTy &Action) { | |||
4705 | Action.Enter(CGF); | |||
4706 | OMPPrivateScope PrivateScope(CGF); | |||
4707 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4708 | (void)PrivateScope.Privatize(); | |||
4709 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, | |||
4710 | CodeGenDistribute); | |||
4711 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4712 | }; | |||
4713 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); | |||
4714 | emitPostUpdateForReductionClause(*this, S, | |||
4715 | [](CodeGenFunction &) { return nullptr; }); | |||
4716 | } | |||
4717 | ||||
4718 | void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( | |||
4719 | const OMPTeamsDistributeParallelForSimdDirective &S) { | |||
4720 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4721 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
4722 | S.getDistInc()); | |||
4723 | }; | |||
4724 | ||||
4725 | // Emit teams region as a standalone region. | |||
4726 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4727 | PrePostActionTy &Action) { | |||
4728 | Action.Enter(CGF); | |||
4729 | OMPPrivateScope PrivateScope(CGF); | |||
4730 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4731 | (void)PrivateScope.Privatize(); | |||
4732 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( | |||
4733 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); | |||
4734 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4735 | }; | |||
4736 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd, | |||
4737 | CodeGen); | |||
4738 | emitPostUpdateForReductionClause(*this, S, | |||
4739 | [](CodeGenFunction &) { return nullptr; }); | |||
4740 | } | |||
4741 | ||||
4742 | static void emitTargetTeamsDistributeParallelForRegion( | |||
4743 | CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, | |||
4744 | PrePostActionTy &Action) { | |||
4745 | Action.Enter(CGF); | |||
4746 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4747 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
4748 | S.getDistInc()); | |||
4749 | }; | |||
4750 | ||||
4751 | // Emit teams region as a standalone region. | |||
4752 | auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4753 | PrePostActionTy &Action) { | |||
4754 | Action.Enter(CGF); | |||
4755 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4756 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4757 | (void)PrivateScope.Privatize(); | |||
4758 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( | |||
4759 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); | |||
4760 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4761 | }; | |||
4762 | ||||
4763 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, | |||
4764 | CodeGenTeams); | |||
4765 | emitPostUpdateForReductionClause(CGF, S, | |||
4766 | [](CodeGenFunction &) { return nullptr; }); | |||
4767 | } | |||
4768 | ||||
4769 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( | |||
4770 | CodeGenModule &CGM, StringRef ParentName, | |||
4771 | const OMPTargetTeamsDistributeParallelForDirective &S) { | |||
4772 | // Emit SPMD target teams distribute parallel for region as a standalone | |||
4773 | // region. | |||
4774 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4775 | emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); | |||
4776 | }; | |||
4777 | llvm::Function *Fn; | |||
4778 | llvm::Constant *Addr; | |||
4779 | // Emit target region as a standalone region. | |||
4780 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4781 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4782 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4782, __PRETTY_FUNCTION__)); | |||
4783 | } | |||
4784 | ||||
4785 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( | |||
4786 | const OMPTargetTeamsDistributeParallelForDirective &S) { | |||
4787 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4788 | emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); | |||
4789 | }; | |||
4790 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4791 | } | |||
4792 | ||||
4793 | static void emitTargetTeamsDistributeParallelForSimdRegion( | |||
4794 | CodeGenFunction &CGF, | |||
4795 | const OMPTargetTeamsDistributeParallelForSimdDirective &S, | |||
4796 | PrePostActionTy &Action) { | |||
4797 | Action.Enter(CGF); | |||
4798 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4799 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, | |||
4800 | S.getDistInc()); | |||
4801 | }; | |||
4802 | ||||
4803 | // Emit teams region as a standalone region. | |||
4804 | auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, | |||
4805 | PrePostActionTy &Action) { | |||
4806 | Action.Enter(CGF); | |||
4807 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
4808 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
4809 | (void)PrivateScope.Privatize(); | |||
4810 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( | |||
4811 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); | |||
4812 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); | |||
4813 | }; | |||
4814 | ||||
4815 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, | |||
4816 | CodeGenTeams); | |||
4817 | emitPostUpdateForReductionClause(CGF, S, | |||
4818 | [](CodeGenFunction &) { return nullptr; }); | |||
4819 | } | |||
4820 | ||||
4821 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( | |||
4822 | CodeGenModule &CGM, StringRef ParentName, | |||
4823 | const OMPTargetTeamsDistributeParallelForSimdDirective &S) { | |||
4824 | // Emit SPMD target teams distribute parallel for simd region as a standalone | |||
4825 | // region. | |||
4826 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4827 | emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); | |||
4828 | }; | |||
4829 | llvm::Function *Fn; | |||
4830 | llvm::Constant *Addr; | |||
4831 | // Emit target region as a standalone region. | |||
4832 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
4833 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
4834 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4834, __PRETTY_FUNCTION__)); | |||
4835 | } | |||
4836 | ||||
4837 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( | |||
4838 | const OMPTargetTeamsDistributeParallelForSimdDirective &S) { | |||
4839 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4840 | emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); | |||
4841 | }; | |||
4842 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
4843 | } | |||
4844 | ||||
4845 | void CodeGenFunction::EmitOMPCancellationPointDirective( | |||
4846 | const OMPCancellationPointDirective &S) { | |||
4847 | CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(), | |||
4848 | S.getCancelRegion()); | |||
4849 | } | |||
4850 | ||||
4851 | void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { | |||
4852 | const Expr *IfCond = nullptr; | |||
4853 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
4854 | if (C->getNameModifier() == OMPD_unknown || | |||
4855 | C->getNameModifier() == OMPD_cancel) { | |||
4856 | IfCond = C->getCondition(); | |||
4857 | break; | |||
4858 | } | |||
4859 | } | |||
4860 | if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) { | |||
4861 | // TODO: This check is necessary as we only generate `omp parallel` through | |||
4862 | // the OpenMPIRBuilder for now. | |||
4863 | if (S.getCancelRegion() == OMPD_parallel) { | |||
4864 | llvm::Value *IfCondition = nullptr; | |||
4865 | if (IfCond) | |||
4866 | IfCondition = EmitScalarExpr(IfCond, | |||
4867 | /*IgnoreResultAssign=*/true); | |||
4868 | return Builder.restoreIP( | |||
4869 | OMPBuilder->CreateCancel(Builder, IfCondition, S.getCancelRegion())); | |||
4870 | } | |||
4871 | } | |||
4872 | ||||
4873 | CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond, | |||
4874 | S.getCancelRegion()); | |||
4875 | } | |||
4876 | ||||
4877 | CodeGenFunction::JumpDest | |||
4878 | CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { | |||
4879 | if (Kind == OMPD_parallel || Kind == OMPD_task || | |||
4880 | Kind == OMPD_target_parallel) | |||
4881 | return ReturnBlock; | |||
4882 | assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)) | |||
4883 | Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)) | |||
4884 | Kind == OMPD_distribute_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)) | |||
4885 | Kind == OMPD_target_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)) | |||
4886 | Kind == OMPD_teams_distribute_parallel_for ||((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)) | |||
4887 | Kind == OMPD_target_teams_distribute_parallel_for)((Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for ) ? static_cast<void> (0) : __assert_fail ("Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4887, __PRETTY_FUNCTION__)); | |||
4888 | return OMPCancelStack.getExitBlock(); | |||
4889 | } | |||
4890 | ||||
4891 | void CodeGenFunction::EmitOMPUseDevicePtrClause( | |||
4892 | const OMPClause &NC, OMPPrivateScope &PrivateScope, | |||
4893 | const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) { | |||
4894 | const auto &C = cast<OMPUseDevicePtrClause>(NC); | |||
4895 | auto OrigVarIt = C.varlist_begin(); | |||
4896 | auto InitIt = C.inits().begin(); | |||
4897 | for (const Expr *PvtVarIt : C.private_copies()) { | |||
4898 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl()); | |||
4899 | const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl()); | |||
4900 | const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl()); | |||
4901 | ||||
4902 | // In order to identify the right initializer we need to match the | |||
4903 | // declaration used by the mapping logic. In some cases we may get | |||
4904 | // OMPCapturedExprDecl that refers to the original declaration. | |||
4905 | const ValueDecl *MatchingVD = OrigVD; | |||
4906 | if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) { | |||
4907 | // OMPCapturedExprDecl are used to privative fields of the current | |||
4908 | // structure. | |||
4909 | const auto *ME = cast<MemberExpr>(OED->getInit()); | |||
4910 | assert(isa<CXXThisExpr>(ME->getBase()) &&((isa<CXXThisExpr>(ME->getBase()) && "Base should be the current struct!" ) ? static_cast<void> (0) : __assert_fail ("isa<CXXThisExpr>(ME->getBase()) && \"Base should be the current struct!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4911, __PRETTY_FUNCTION__)) | |||
4911 | "Base should be the current struct!")((isa<CXXThisExpr>(ME->getBase()) && "Base should be the current struct!" ) ? static_cast<void> (0) : __assert_fail ("isa<CXXThisExpr>(ME->getBase()) && \"Base should be the current struct!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4911, __PRETTY_FUNCTION__)); | |||
4912 | MatchingVD = ME->getMemberDecl(); | |||
4913 | } | |||
4914 | ||||
4915 | // If we don't have information about the current list item, move on to | |||
4916 | // the next one. | |||
4917 | auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD); | |||
4918 | if (InitAddrIt == CaptureDeviceAddrMap.end()) | |||
4919 | continue; | |||
4920 | ||||
4921 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD, | |||
4922 | InitAddrIt, InitVD, | |||
4923 | PvtVD]() { | |||
4924 | // Initialize the temporary initialization variable with the address we | |||
4925 | // get from the runtime library. We have to cast the source address | |||
4926 | // because it is always a void *. References are materialized in the | |||
4927 | // privatization scope, so the initialization here disregards the fact | |||
4928 | // the original variable is a reference. | |||
4929 | QualType AddrQTy = | |||
4930 | getContext().getPointerType(OrigVD->getType().getNonReferenceType()); | |||
4931 | llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy); | |||
4932 | Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy); | |||
4933 | setAddrOfLocalVar(InitVD, InitAddr); | |||
4934 | ||||
4935 | // Emit private declaration, it will be initialized by the value we | |||
4936 | // declaration we just added to the local declarations map. | |||
4937 | EmitDecl(*PvtVD); | |||
4938 | ||||
4939 | // The initialization variables reached its purpose in the emission | |||
4940 | // of the previous declaration, so we don't need it anymore. | |||
4941 | LocalDeclMap.erase(InitVD); | |||
4942 | ||||
4943 | // Return the address of the private variable. | |||
4944 | return GetAddrOfLocalVar(PvtVD); | |||
4945 | }); | |||
4946 | assert(IsRegistered && "firstprivate var already registered as private")((IsRegistered && "firstprivate var already registered as private" ) ? static_cast<void> (0) : __assert_fail ("IsRegistered && \"firstprivate var already registered as private\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 4946, __PRETTY_FUNCTION__)); | |||
4947 | // Silence the warning about unused variable. | |||
4948 | (void)IsRegistered; | |||
4949 | ||||
4950 | ++OrigVarIt; | |||
4951 | ++InitIt; | |||
4952 | } | |||
4953 | } | |||
4954 | ||||
4955 | // Generate the instructions for '#pragma omp target data' directive. | |||
4956 | void CodeGenFunction::EmitOMPTargetDataDirective( | |||
4957 | const OMPTargetDataDirective &S) { | |||
4958 | CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true); | |||
4959 | ||||
4960 | // Create a pre/post action to signal the privatization of the device pointer. | |||
4961 | // This action can be replaced by the OpenMP runtime code generation to | |||
4962 | // deactivate privatization. | |||
4963 | bool PrivatizeDevicePointers = false; | |||
4964 | class DevicePointerPrivActionTy : public PrePostActionTy { | |||
4965 | bool &PrivatizeDevicePointers; | |||
4966 | ||||
4967 | public: | |||
4968 | explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers) | |||
4969 | : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {} | |||
4970 | void Enter(CodeGenFunction &CGF) override { | |||
4971 | PrivatizeDevicePointers = true; | |||
4972 | } | |||
4973 | }; | |||
4974 | DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers); | |||
4975 | ||||
4976 | auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers]( | |||
4977 | CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
4978 | auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
4979 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); | |||
4980 | }; | |||
4981 | ||||
4982 | // Codegen that selects whether to generate the privatization code or not. | |||
4983 | auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers, | |||
4984 | &InnermostCodeGen](CodeGenFunction &CGF, | |||
4985 | PrePostActionTy &Action) { | |||
4986 | RegionCodeGenTy RCG(InnermostCodeGen); | |||
4987 | PrivatizeDevicePointers = false; | |||
4988 | ||||
4989 | // Call the pre-action to change the status of PrivatizeDevicePointers if | |||
4990 | // needed. | |||
4991 | Action.Enter(CGF); | |||
4992 | ||||
4993 | if (PrivatizeDevicePointers) { | |||
4994 | OMPPrivateScope PrivateScope(CGF); | |||
4995 | // Emit all instances of the use_device_ptr clause. | |||
4996 | for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>()) | |||
4997 | CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, | |||
4998 | Info.CaptureDeviceAddrMap); | |||
4999 | (void)PrivateScope.Privatize(); | |||
5000 | RCG(CGF); | |||
5001 | } else { | |||
5002 | RCG(CGF); | |||
5003 | } | |||
5004 | }; | |||
5005 | ||||
5006 | // Forward the provided action to the privatization codegen. | |||
5007 | RegionCodeGenTy PrivRCG(PrivCodeGen); | |||
5008 | PrivRCG.setAction(Action); | |||
5009 | ||||
5010 | // Notwithstanding the body of the region is emitted as inlined directive, | |||
5011 | // we don't use an inline scope as changes in the references inside the | |||
5012 | // region are expected to be visible outside, so we do not privative them. | |||
5013 | OMPLexicalScope Scope(CGF, S); | |||
5014 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, | |||
5015 | PrivRCG); | |||
5016 | }; | |||
5017 | ||||
5018 | RegionCodeGenTy RCG(CodeGen); | |||
5019 | ||||
5020 | // If we don't have target devices, don't bother emitting the data mapping | |||
5021 | // code. | |||
5022 | if (CGM.getLangOpts().OMPTargetTriples.empty()) { | |||
5023 | RCG(*this); | |||
5024 | return; | |||
5025 | } | |||
5026 | ||||
5027 | // Check if we have any if clause associated with the directive. | |||
5028 | const Expr *IfCond = nullptr; | |||
5029 | if (const auto *C = S.getSingleClause<OMPIfClause>()) | |||
5030 | IfCond = C->getCondition(); | |||
5031 | ||||
5032 | // Check if we have any device clause associated with the directive. | |||
5033 | const Expr *Device = nullptr; | |||
5034 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) | |||
5035 | Device = C->getDevice(); | |||
5036 | ||||
5037 | // Set the action to signal privatization of device pointers. | |||
5038 | RCG.setAction(PrivAction); | |||
5039 | ||||
5040 | // Emit region code. | |||
5041 | CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG, | |||
5042 | Info); | |||
5043 | } | |||
5044 | ||||
5045 | void CodeGenFunction::EmitOMPTargetEnterDataDirective( | |||
5046 | const OMPTargetEnterDataDirective &S) { | |||
5047 | // If we don't have target devices, don't bother emitting the data mapping | |||
5048 | // code. | |||
5049 | if (CGM.getLangOpts().OMPTargetTriples.empty()) | |||
5050 | return; | |||
5051 | ||||
5052 | // Check if we have any if clause associated with the directive. | |||
5053 | const Expr *IfCond = nullptr; | |||
5054 | if (const auto *C = S.getSingleClause<OMPIfClause>()) | |||
5055 | IfCond = C->getCondition(); | |||
5056 | ||||
5057 | // Check if we have any device clause associated with the directive. | |||
5058 | const Expr *Device = nullptr; | |||
5059 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) | |||
5060 | Device = C->getDevice(); | |||
5061 | ||||
5062 | OMPLexicalScope Scope(*this, S, OMPD_task); | |||
5063 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); | |||
5064 | } | |||
5065 | ||||
5066 | void CodeGenFunction::EmitOMPTargetExitDataDirective( | |||
5067 | const OMPTargetExitDataDirective &S) { | |||
5068 | // If we don't have target devices, don't bother emitting the data mapping | |||
5069 | // code. | |||
5070 | if (CGM.getLangOpts().OMPTargetTriples.empty()) | |||
5071 | return; | |||
5072 | ||||
5073 | // Check if we have any if clause associated with the directive. | |||
5074 | const Expr *IfCond = nullptr; | |||
5075 | if (const auto *C = S.getSingleClause<OMPIfClause>()) | |||
5076 | IfCond = C->getCondition(); | |||
5077 | ||||
5078 | // Check if we have any device clause associated with the directive. | |||
5079 | const Expr *Device = nullptr; | |||
5080 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) | |||
5081 | Device = C->getDevice(); | |||
5082 | ||||
5083 | OMPLexicalScope Scope(*this, S, OMPD_task); | |||
5084 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); | |||
5085 | } | |||
5086 | ||||
5087 | static void emitTargetParallelRegion(CodeGenFunction &CGF, | |||
5088 | const OMPTargetParallelDirective &S, | |||
5089 | PrePostActionTy &Action) { | |||
5090 | // Get the captured statement associated with the 'parallel' region. | |||
5091 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); | |||
5092 | Action.Enter(CGF); | |||
5093 | auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5094 | Action.Enter(CGF); | |||
5095 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |||
5096 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); | |||
5097 | CGF.EmitOMPPrivateClause(S, PrivateScope); | |||
5098 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); | |||
5099 | (void)PrivateScope.Privatize(); | |||
5100 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) | |||
5101 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); | |||
5102 | // TODO: Add support for clauses. | |||
5103 | CGF.EmitStmt(CS->getCapturedStmt()); | |||
5104 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); | |||
5105 | }; | |||
5106 | emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, | |||
5107 | emitEmptyBoundParameters); | |||
5108 | emitPostUpdateForReductionClause(CGF, S, | |||
5109 | [](CodeGenFunction &) { return nullptr; }); | |||
5110 | } | |||
5111 | ||||
5112 | void CodeGenFunction::EmitOMPTargetParallelDeviceFunction( | |||
5113 | CodeGenModule &CGM, StringRef ParentName, | |||
5114 | const OMPTargetParallelDirective &S) { | |||
5115 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5116 | emitTargetParallelRegion(CGF, S, Action); | |||
5117 | }; | |||
5118 | llvm::Function *Fn; | |||
5119 | llvm::Constant *Addr; | |||
5120 | // Emit target region as a standalone region. | |||
5121 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
5122 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
5123 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 5123, __PRETTY_FUNCTION__)); | |||
5124 | } | |||
5125 | ||||
5126 | void CodeGenFunction::EmitOMPTargetParallelDirective( | |||
5127 | const OMPTargetParallelDirective &S) { | |||
5128 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5129 | emitTargetParallelRegion(CGF, S, Action); | |||
5130 | }; | |||
5131 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
5132 | } | |||
5133 | ||||
5134 | static void emitTargetParallelForRegion(CodeGenFunction &CGF, | |||
5135 | const OMPTargetParallelForDirective &S, | |||
5136 | PrePostActionTy &Action) { | |||
5137 | Action.Enter(CGF); | |||
5138 | // Emit directive as a combined directive that consists of two implicit | |||
5139 | // directives: 'parallel' with 'for' directive. | |||
5140 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5141 | Action.Enter(CGF); | |||
5142 | CodeGenFunction::OMPCancelStackRAII CancelRegion( | |||
5143 | CGF, OMPD_target_parallel_for, S.hasCancel()); | |||
5144 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, | |||
5145 | emitDispatchForLoopBounds); | |||
5146 | }; | |||
5147 | emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, | |||
5148 | emitEmptyBoundParameters); | |||
5149 | } | |||
5150 | ||||
5151 | void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( | |||
5152 | CodeGenModule &CGM, StringRef ParentName, | |||
5153 | const OMPTargetParallelForDirective &S) { | |||
5154 | // Emit SPMD target parallel for region as a standalone region. | |||
5155 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5156 | emitTargetParallelForRegion(CGF, S, Action); | |||
5157 | }; | |||
5158 | llvm::Function *Fn; | |||
5159 | llvm::Constant *Addr; | |||
5160 | // Emit target region as a standalone region. | |||
5161 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
5162 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
5163 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 5163, __PRETTY_FUNCTION__)); | |||
5164 | } | |||
5165 | ||||
5166 | void CodeGenFunction::EmitOMPTargetParallelForDirective( | |||
5167 | const OMPTargetParallelForDirective &S) { | |||
5168 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5169 | emitTargetParallelForRegion(CGF, S, Action); | |||
5170 | }; | |||
5171 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
5172 | } | |||
5173 | ||||
5174 | static void | |||
5175 | emitTargetParallelForSimdRegion(CodeGenFunction &CGF, | |||
5176 | const OMPTargetParallelForSimdDirective &S, | |||
5177 | PrePostActionTy &Action) { | |||
5178 | Action.Enter(CGF); | |||
5179 | // Emit directive as a combined directive that consists of two implicit | |||
5180 | // directives: 'parallel' with 'for' directive. | |||
5181 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5182 | Action.Enter(CGF); | |||
5183 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, | |||
5184 | emitDispatchForLoopBounds); | |||
5185 | }; | |||
5186 | emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, | |||
5187 | emitEmptyBoundParameters); | |||
5188 | } | |||
5189 | ||||
5190 | void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( | |||
5191 | CodeGenModule &CGM, StringRef ParentName, | |||
5192 | const OMPTargetParallelForSimdDirective &S) { | |||
5193 | // Emit SPMD target parallel for region as a standalone region. | |||
5194 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5195 | emitTargetParallelForSimdRegion(CGF, S, Action); | |||
5196 | }; | |||
5197 | llvm::Function *Fn; | |||
5198 | llvm::Constant *Addr; | |||
5199 | // Emit target region as a standalone region. | |||
5200 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( | |||
5201 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); | |||
5202 | assert(Fn && Addr && "Target device function emission failed.")((Fn && Addr && "Target device function emission failed." ) ? static_cast<void> (0) : __assert_fail ("Fn && Addr && \"Target device function emission failed.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 5202, __PRETTY_FUNCTION__)); | |||
5203 | } | |||
5204 | ||||
5205 | void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( | |||
5206 | const OMPTargetParallelForSimdDirective &S) { | |||
5207 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5208 | emitTargetParallelForSimdRegion(CGF, S, Action); | |||
5209 | }; | |||
5210 | emitCommonOMPTargetDirective(*this, S, CodeGen); | |||
5211 | } | |||
5212 | ||||
5213 | /// Emit a helper variable and return corresponding lvalue. | |||
5214 | static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, | |||
5215 | const ImplicitParamDecl *PVD, | |||
5216 | CodeGenFunction::OMPPrivateScope &Privates) { | |||
5217 | const auto *VDecl = cast<VarDecl>(Helper->getDecl()); | |||
5218 | Privates.addPrivate(VDecl, | |||
5219 | [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); | |||
5220 | } | |||
5221 | ||||
5222 | void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) { | |||
5223 | assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()))((isOpenMPTaskLoopDirective(S.getDirectiveKind())) ? static_cast <void> (0) : __assert_fail ("isOpenMPTaskLoopDirective(S.getDirectiveKind())" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/CodeGen/CGStmtOpenMP.cpp" , 5223, __PRETTY_FUNCTION__)); | |||
5224 | // Emit outlined function for task construct. | |||
5225 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop); | |||
5226 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); | |||
5227 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); | |||
5228 | const Expr *IfCond = nullptr; | |||
5229 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { | |||
5230 | if (C->getNameModifier() == OMPD_unknown || | |||
5231 | C->getNameModifier() == OMPD_taskloop) { | |||
5232 | IfCond = C->getCondition(); | |||
5233 | break; | |||
5234 | } | |||
5235 | } | |||
5236 | ||||
5237 | OMPTaskDataTy Data; | |||
5238 | // Check if taskloop must be emitted without taskgroup. | |||
5239 | Data.Nogroup = S.getSingleClause<OMPNogroupClause>(); | |||
5240 | // TODO: Check if we should emit tied or untied task. | |||
5241 | Data.Tied = true; | |||
5242 | // Set scheduling for taskloop | |||
5243 | if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) { | |||
5244 | // grainsize clause | |||
5245 | Data.Schedule.setInt(/*IntVal=*/false); | |||
5246 | Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize())); | |||
5247 | } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) { | |||
5248 | // num_tasks clause | |||
5249 | Data.Schedule.setInt(/*IntVal=*/true); | |||
5250 | Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks())); | |||
5251 | } | |||
5252 | ||||
5253 | auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
5254 | // if (PreCond) { | |||
5255 | // for (IV in 0..LastIteration) BODY; | |||
5256 | // <Final counter/linear vars updates>; | |||
5257 | // } | |||
5258 | // | |||
5259 | ||||
5260 | // Emit: if (PreCond) - begin. | |||
5261 | // If the condition constant folds and can be elided, avoid emitting the | |||
5262 | // whole loop. | |||
5263 | bool CondConstant; | |||
5264 | llvm::BasicBlock *ContBlock = nullptr; | |||
5265 | OMPLoopScope PreInitScope(CGF, S); | |||
5266 | if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { | |||
5267 | if (!CondConstant) | |||
5268 | return; | |||
5269 | } else { | |||
5270 | llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); | |||
5271 | ContBlock = CGF.createBasicBlock("taskloop.if.end"); | |||
5272 | emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, | |||
5273 | CGF.getProfileCount(&S)); | |||
5274 | CGF.EmitBlock(ThenBlock); | |||
5275 | CGF.incrementProfileCounter(&S); | |||
5276 | } | |||
5277 | ||||
5278 | (void)CGF.EmitOMPLinearClauseInit(S); | |||
5279 | ||||
5280 | OMPPrivateScope LoopScope(CGF); | |||
5281 | // Emit helper vars inits. | |||
5282 | enum { LowerBound = 5, UpperBound, Stride, LastIter }; | |||
5283 | auto *I = CS->getCapturedDecl()->param_begin(); | |||
5284 | auto *LBP = std::next(I, LowerBound); | |||
5285 | auto *UBP = std::next(I, UpperBound); | |||
5286 | auto *STP = std::next(I, Stride); | |||
5287 | auto *LIP = std::next(I, LastIter); | |||
5288 | mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, | |||
5289 | LoopScope); | |||
5290 | mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, | |||
5291 | LoopScope); | |||
5292 | mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); | |||
5293 | mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, | |||
5294 | LoopScope); | |||
5295 | CGF.EmitOMPPrivateLoopCounters(S, LoopScope); | |||
5296 | CGF.EmitOMPLinearClause(S, LoopScope); | |||
5297 | bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); | |||
5298 | (void)LoopScope.Privatize(); | |||
5299 | // Emit the loop iteration variable. | |||
5300 | const Expr *IVExpr = S.getIterationVariable(); | |||
5301 | const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); | |||
5302 | CGF.EmitVarDecl(*IVDecl); | |||
5303 | CGF.EmitIgnoredExpr(S.getInit()); | |||
5304 | ||||
5305 | // Emit the iterations count variable. | |||
5306 | // If it is not a variable, Sema decided to calculate iterations count on | |||
5307 | // each iteration (e.g., it is foldable into a constant). | |||
5308 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { | |||
5309 | CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); | |||
5310 | // Emit calculation of the iterations count. | |||
5311 | CGF.EmitIgnoredExpr(S.getCalcLastIteration()); | |||
5312 | } | |||
5313 | ||||
5314 | { | |||
5315 | OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); | |||
5316 | emitCommonSimdLoop( | |||
5317 | CGF, S, | |||
5318 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { | |||
5319 | if (isOpenMPSimdDirective(S.getDirectiveKind())) | |||
5320 | CGF.EmitOMPSimdInit(S); | |||
5321 | }, | |||
5322 | [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { | |||
5323 | CGF.EmitOMPInnerLoop( | |||
5324 | S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), | |||
5325 | [&S](CodeGenFunction &CGF) { | |||
5326 | CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); | |||
5327 | CGF.EmitStopPoint(&S); | |||
5328 | }, | |||
5329 | [](CodeGenFunction &) {}); | |||
5330 | }); | |||
5331 | } | |||
5332 | // Emit: if (PreCond) - end. | |||
5333 | if (ContBlock) { | |||
5334 | CGF.EmitBranch(ContBlock); | |||
5335 | CGF.EmitBlock(ContBlock, true); | |||
5336 | } | |||
5337 | // Emit final copy of the lastprivate variables if IsLastIter != 0. | |||
5338 | if (HasLastprivateClause) { | |||
5339 | CGF.EmitOMPLastprivateClauseFinal( | |||
5340 | S, isOpenMPSimdDirective(S.getDirectiveKind()), | |||
5341 | CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( | |||
5342 | CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, | |||
5343 | (*LIP)->getType(), S.getBeginLoc()))); | |||
5344 | } | |||
5345 | CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { | |||
5346 | return CGF.Builder.CreateIsNotNull( | |||
5347 | CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, | |||
5348 | (*LIP)->getType(), S.getBeginLoc())); | |||
5349 | }); | |||
5350 | }; | |||
5351 | auto &&TaskGen = [&S, SharedsTy, CapturedStruct, | |||
5352 | IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, | |||
5353 | const OMPTaskDataTy &Data) { | |||
5354 | auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, | |||
5355 | &Data](CodeGenFunction &CGF, PrePostActionTy &) { | |||
5356 | OMPLoopScope PreInitScope(CGF, S); | |||
5357 | CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, | |||
5358 | OutlinedFn, SharedsTy, | |||
5359 | CapturedStruct, IfCond, Data); | |||
5360 | }; | |||
5361 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, | |||
5362 | CodeGen); | |||
5363 | }; | |||
5364 | if (Data.Nogroup) { | |||
5365 | EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); | |||
5366 | } else { | |||
5367 | CGM.getOpenMPRuntime().emitTaskgroupRegion( | |||
5368 | *this, | |||
5369 | [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, | |||
5370 | PrePostActionTy &Action) { | |||
5371 | Action.Enter(CGF); | |||
5372 | CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, | |||
5373 | Data); | |||
5374 | }, | |||
5375 | S.getBeginLoc()); | |||
5376 | } | |||
5377 | } | |||
5378 | ||||
5379 | void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) { | |||
5380 | EmitOMPTaskLoopBasedDirective(S); | |||
5381 | } | |||
5382 | ||||
5383 | void CodeGenFunction::EmitOMPTaskLoopSimdDirective( | |||
5384 | const OMPTaskLoopSimdDirective &S) { | |||
5385 | OMPLexicalScope Scope(*this, S); | |||
5386 | EmitOMPTaskLoopBasedDirective(S); | |||
5387 | } | |||
5388 | ||||
5389 | void CodeGenFunction::EmitOMPMasterTaskLoopDirective( | |||
5390 | const OMPMasterTaskLoopDirective &S) { | |||
5391 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5392 | Action.Enter(CGF); | |||
5393 | EmitOMPTaskLoopBasedDirective(S); | |||
5394 | }; | |||
5395 | OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false); | |||
5396 | CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc()); | |||
5397 | } | |||
5398 | ||||
5399 | void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective( | |||
5400 | const OMPMasterTaskLoopSimdDirective &S) { | |||
5401 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5402 | Action.Enter(CGF); | |||
5403 | EmitOMPTaskLoopBasedDirective(S); | |||
5404 | }; | |||
5405 | OMPLexicalScope Scope(*this, S); | |||
5406 | CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc()); | |||
5407 | } | |||
5408 | ||||
5409 | void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective( | |||
5410 | const OMPParallelMasterTaskLoopDirective &S) { | |||
5411 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5412 | auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, | |||
5413 | PrePostActionTy &Action) { | |||
5414 | Action.Enter(CGF); | |||
5415 | CGF.EmitOMPTaskLoopBasedDirective(S); | |||
5416 | }; | |||
5417 | OMPLexicalScope Scope(CGF, S, llvm::None, /*EmitPreInitStmt=*/false); | |||
5418 | CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, | |||
5419 | S.getBeginLoc()); | |||
5420 | }; | |||
5421 | emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen, | |||
5422 | emitEmptyBoundParameters); | |||
5423 | } | |||
5424 | ||||
5425 | void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective( | |||
5426 | const OMPParallelMasterTaskLoopSimdDirective &S) { | |||
5427 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5428 | auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, | |||
5429 | PrePostActionTy &Action) { | |||
5430 | Action.Enter(CGF); | |||
5431 | CGF.EmitOMPTaskLoopBasedDirective(S); | |||
5432 | }; | |||
5433 | OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); | |||
5434 | CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, | |||
5435 | S.getBeginLoc()); | |||
5436 | }; | |||
5437 | emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen, | |||
5438 | emitEmptyBoundParameters); | |||
5439 | } | |||
5440 | ||||
5441 | // Generate the instructions for '#pragma omp target update' directive. | |||
5442 | void CodeGenFunction::EmitOMPTargetUpdateDirective( | |||
5443 | const OMPTargetUpdateDirective &S) { | |||
5444 | // If we don't have target devices, don't bother emitting the data mapping | |||
5445 | // code. | |||
5446 | if (CGM.getLangOpts().OMPTargetTriples.empty()) | |||
5447 | return; | |||
5448 | ||||
5449 | // Check if we have any if clause associated with the directive. | |||
5450 | const Expr *IfCond = nullptr; | |||
5451 | if (const auto *C = S.getSingleClause<OMPIfClause>()) | |||
5452 | IfCond = C->getCondition(); | |||
5453 | ||||
5454 | // Check if we have any device clause associated with the directive. | |||
5455 | const Expr *Device = nullptr; | |||
5456 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) | |||
5457 | Device = C->getDevice(); | |||
5458 | ||||
5459 | OMPLexicalScope Scope(*this, S, OMPD_task); | |||
5460 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); | |||
5461 | } | |||
5462 | ||||
5463 | void CodeGenFunction::EmitSimpleOMPExecutableDirective( | |||
5464 | const OMPExecutableDirective &D) { | |||
5465 | if (!D.hasAssociatedStmt() || !D.getAssociatedStmt()) | |||
5466 | return; | |||
5467 | auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { | |||
5468 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { | |||
| ||||
5469 | emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); | |||
5470 | } else { | |||
5471 | OMPPrivateScope LoopGlobals(CGF); | |||
5472 | if (const auto *LD
| |||
5473 | for (const Expr *E : LD->counters()) { | |||
5474 | const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); | |||
5475 | if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { | |||
| ||||
5476 | LValue GlobLVal = CGF.EmitLValue(E); | |||
5477 | LoopGlobals.addPrivate( | |||
5478 | VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); | |||
5479 | } | |||
5480 | if (isa<OMPCapturedExprDecl>(VD)) { | |||
5481 | // Emit only those that were not explicitly referenced in clauses. | |||
5482 | if (!CGF.LocalDeclMap.count(VD)) | |||
5483 | CGF.EmitVarDecl(*VD); | |||
5484 | } | |||
5485 | } | |||
5486 | for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) { | |||
5487 | if (!C->getNumForLoops()) | |||
5488 | continue; | |||
5489 | for (unsigned I = LD->getCollapsedNumber(), | |||
5490 | E = C->getLoopNumIterations().size(); | |||
5491 | I < E; ++I) { | |||
5492 | if (const auto *VD = dyn_cast<OMPCapturedExprDecl>( | |||
5493 | cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) { | |||
5494 | // Emit only those that were not explicitly referenced in clauses. | |||
5495 | if (!CGF.LocalDeclMap.count(VD)) | |||
5496 | CGF.EmitVarDecl(*VD); | |||
5497 | } | |||
5498 | } | |||
5499 | } | |||
5500 | } | |||
5501 | LoopGlobals.Privatize(); | |||
5502 | CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); | |||
5503 | } | |||
5504 | }; | |||
5505 | OMPSimdLexicalScope Scope(*this, D); | |||
5506 | CGM.getOpenMPRuntime().emitInlinedDirective( | |||
5507 | *this, | |||
5508 | isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd | |||
5509 | : D.getDirectiveKind(), | |||
5510 | CodeGen); | |||
5511 | } |